mBert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mBert.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMkKvvZjXOKK9T3sUEX+cWY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/emarkou/multilingual-bert-text-classification/blob/master/mBert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iorCIFLWP3S4",
        "outputId": "606381cf-3a23-4a27-81fd-b16fd0d9f9f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: tokenizers==0.8.1.rc2 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJUnDFzgzC3q"
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, SequentialSampler\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig, BertPreTrainedModel, BertModel\n",
        "from transformers import get_linear_schedule_with_warmup"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcalmCCuTwcu",
        "outputId": "cff69fab-797a-40c3-f84b-df4289b77919",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name == '/device:GPU:0':\n",
        "    print(f'Found GPU at: {device_name}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLZLsS3fUli7",
        "outputId": "02bb051f-81a4-4ee9-cdbf-456f602087ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('GPU in use:', torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    print('using the CPU')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU in use: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aZUKWVkRBrQ"
      },
      "source": [
        "MAX_LEN = 128 # max sequences length\n",
        "batch_size = 32"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IokttCXpR1_"
      },
      "source": [
        "# extra preprocessing steps\n",
        "# prepend CLS and append SEP, truncate, pad\n",
        "\n",
        "labels_encoding = {\n",
        "    \"CCAT\": 0,\n",
        "    \"ECAT\": 1,\n",
        "    \"GCAT\": 2,\n",
        "    \"MCAT\": 3\n",
        "}\n",
        "\n",
        "def preprocessing(df):\n",
        "    sentences = df.sentence.values\n",
        "    labels = np.array([labels_encoding[l] for l in df.label.values])\n",
        "\n",
        "    tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=True)\n",
        "    \n",
        "    encoded_sentences = []\n",
        "    for sent in sentences:\n",
        "        encoded_sent = tokenizer.encode(\n",
        "                            sent,\n",
        "                            add_special_tokens = True,\n",
        "                            truncation=True,\n",
        "                            max_length = MAX_LEN\n",
        "                    )\n",
        "        \n",
        "        encoded_sentences.append(encoded_sent)\n",
        "    encoded_sentences = pad_sequences(encoded_sentences, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                            value=0, truncating=\"post\", padding=\"post\")\n",
        "    return encoded_sentences, labels\n",
        "    \n",
        "def attention_masks(encoded_sentences):\n",
        "    # attention masks, 0 for padding, 1 for actual token\n",
        "    attention_masks = []\n",
        "    for sent in encoded_sentences:\n",
        "        att_mask = [int(token_id > 0) for token_id in sent]\n",
        "        attention_masks.append(att_mask)\n",
        "    return attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRUoG8ZeU8xz"
      },
      "source": [
        "# load the datasets\n",
        "df = pd.read_csv(\"./english.train.10000\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "df_test = pd.read_csv(\"./english.dev\", delimiter='\\t', header=None, names=['label', 'sentence'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rm_VIbP8QRtk",
        "outputId": "78f906f5-d7ac-48a0-9d06-d7c943705fce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ECAT</td>\n",
              "      <td>France could make more tax cuts than planned i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ECAT</td>\n",
              "      <td>Ryan, Beck &amp; Co said it won $2.679 million of ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>MCAT</td>\n",
              "      <td>Brazilian shares plummeted more than 3.0 perce...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>GCAT</td>\n",
              "      <td>Boris Becker said after beating Jan Siemerink ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GCAT</td>\n",
              "      <td>Leading stories in the Greek financial press: ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  label                                           sentence\n",
              "0  ECAT  France could make more tax cuts than planned i...\n",
              "1  ECAT  Ryan, Beck & Co said it won $2.679 million of ...\n",
              "2  MCAT  Brazilian shares plummeted more than 3.0 perce...\n",
              "3  GCAT  Boris Becker said after beating Jan Siemerink ...\n",
              "4  GCAT  Leading stories in the Greek financial press: ..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3FStGvIpttG"
      },
      "source": [
        "train_encoded_sentences, train_labels = preprocessing(df)\n",
        "train_attention_masks = attention_masks(train_encoded_sentences)\n",
        "\n",
        "test_encoded_sentences, test_labels = preprocessing(df_test)\n",
        "test_attention_masks = attention_masks(test_encoded_sentences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XsqHZ2xIpN9K"
      },
      "source": [
        "train_inputs = torch.tensor(train_encoded_sentences)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_attention_masks)\n",
        "\n",
        "validation_inputs = torch.tensor(test_encoded_sentences)\n",
        "validation_labels = torch.tensor(test_labels)\n",
        "validation_masks = torch.tensor(test_attention_masks)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LoJ-dReQxFiC"
      },
      "source": [
        "# data loader for training\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = SequentialSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# data loader for validation\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xON2LLwOxpiB",
        "outputId": "364b3e6a-c845-4bb0-ba22-518d456dc8fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "\n",
        "import random\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-multilingual-cased\",\n",
        "    num_labels = 4,   \n",
        "    output_attentions = False, \n",
        "    output_hidden_states = False, \n",
        ")\n",
        "\n",
        "model.cuda()\n",
        "\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 3e-5, \n",
        "                  eps = 1e-8, \n",
        "                  weight_decay = 0.01\n",
        "                )\n",
        "\n",
        "epochs = 3\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # 10% * datasetSize/batchSize\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-multilingual-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IZL4pCBWMH6E"
      },
      "source": [
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "def compute_accuracy(preds, labels):\n",
        "    p = np.argmax(preds, axis=1).flatten()\n",
        "    l = labels.flatten()\n",
        "    return np.sum(p==l)/len(l)\n",
        "\n",
        "def run_train(epochs):\n",
        "    losses = []\n",
        "    for e in range(epochs):\n",
        "        print('======== Epoch {:} / {:} ========'.format(e + 1, epochs))\n",
        "        start_train_time = time.time()\n",
        "        total_loss = 0\n",
        "        model.train()\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "            if step%10 == 0:\n",
        "                elapsed = time.time()-start_train_time\n",
        "                print(f'{step}/{len(train_dataloader)} --> Time elapsed {elapsed}')\n",
        "\n",
        "            # input_data, input_masks, input_labels = batch\n",
        "            input_data = batch[0].to(device)\n",
        "            input_masks = batch[1].to(device)\n",
        "            input_labels = batch[2].to(device)\n",
        "\n",
        "            model.zero_grad()\n",
        "\n",
        "            # forward propagation\n",
        "            out = model(input_data,\n",
        "                        token_type_ids = None, \n",
        "                        attention_mask = input_masks,\n",
        "                        labels = input_labels)\n",
        "            \n",
        "            loss = out[0]\n",
        "            total_loss = total_loss + loss.item()\n",
        "\n",
        "            # backward propagation\n",
        "            loss.backward()\n",
        "            \n",
        "            torch.nn.utils.clip_grad_norm(model.parameters(), 1)\n",
        "\n",
        "            optimizer.step()\n",
        "        \n",
        "        epoch_loss = total_loss/len(train_dataloader)\n",
        "        losses.append(epoch_loss)\n",
        "        print(f\"Training took {time.time()-start_train_time}\")\n",
        "\n",
        "        # Validation\n",
        "        start_validation_time = time.time()\n",
        "        model.eval()\n",
        "        eval_loss, eval_acc = 0,0\n",
        "        for step, batch in enumerate(validation_dataloader):\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "            eval_data, eval_masks, eval_labels = batch\n",
        "            with torch.no_grad():\n",
        "                out = model(eval_data,\n",
        "                            token_type_ids = None, \n",
        "                            attention_mask=eval_masks)\n",
        "            logits = out[0]\n",
        "\n",
        "            #  Uncomment for GPU execution\n",
        "            logits = logits.detach().cpu().numpy()\n",
        "            eval_labels = eval_labels.to('cpu').numpy()\n",
        "            batch_acc = compute_accuracy(logits, eval_labels)\n",
        "\n",
        "            # Uncomment for CPU execution\n",
        "            # batch_acc = compute_accuracy(logits.numpy(), eval_labels.numpy())\n",
        "\n",
        "            eval_acc += batch_acc\n",
        "        print(f\"Accuracy: {eval_acc/(step+1)}, Time elapsed: {time.time()-start_validation_time}\")\n",
        "    return losses\n",
        "            "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAILAFFrTFhq",
        "outputId": "34ecefa6-28ee-4ecb-eedb-8196a8406a2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "losses = run_train(epochs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "======== Epoch 1 / 3 ========\n",
            "0/313 --> Time elapsed 0.001956462860107422\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:40: UserWarning: torch.nn.utils.clip_grad_norm is now deprecated in favor of torch.nn.utils.clip_grad_norm_.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10/313 --> Time elapsed 6.9794535636901855\n",
            "20/313 --> Time elapsed 14.00432801246643\n",
            "30/313 --> Time elapsed 21.007280588150024\n",
            "40/313 --> Time elapsed 28.093219757080078\n",
            "50/313 --> Time elapsed 35.22671961784363\n",
            "60/313 --> Time elapsed 42.34084105491638\n",
            "70/313 --> Time elapsed 49.51961612701416\n",
            "80/313 --> Time elapsed 56.761051654815674\n",
            "90/313 --> Time elapsed 64.05973291397095\n",
            "100/313 --> Time elapsed 71.35890173912048\n",
            "110/313 --> Time elapsed 78.66858291625977\n",
            "120/313 --> Time elapsed 86.04460859298706\n",
            "130/313 --> Time elapsed 93.50694799423218\n",
            "140/313 --> Time elapsed 101.0357723236084\n",
            "150/313 --> Time elapsed 108.52301597595215\n",
            "160/313 --> Time elapsed 116.03029012680054\n",
            "170/313 --> Time elapsed 123.58999395370483\n",
            "180/313 --> Time elapsed 131.18279719352722\n",
            "190/313 --> Time elapsed 138.8018410205841\n",
            "200/313 --> Time elapsed 146.473863363266\n",
            "210/313 --> Time elapsed 154.1548933982849\n",
            "220/313 --> Time elapsed 161.85852766036987\n",
            "230/313 --> Time elapsed 169.602641582489\n",
            "240/313 --> Time elapsed 177.40800762176514\n",
            "250/313 --> Time elapsed 185.22651553153992\n",
            "260/313 --> Time elapsed 193.10465025901794\n",
            "270/313 --> Time elapsed 200.99086689949036\n",
            "280/313 --> Time elapsed 208.9062705039978\n",
            "290/313 --> Time elapsed 216.8382601737976\n",
            "300/313 --> Time elapsed 224.78997588157654\n",
            "310/313 --> Time elapsed 232.74318313598633\n",
            "Training took 234.78063344955444\n",
            "Accuracy: 0.9658203125, Time elapsed: 8.015210628509521\n",
            "======== Epoch 2 / 3 ========\n",
            "0/313 --> Time elapsed 0.0018017292022705078\n",
            "10/313 --> Time elapsed 7.913478374481201\n",
            "20/313 --> Time elapsed 15.819361448287964\n",
            "30/313 --> Time elapsed 23.724483728408813\n",
            "40/313 --> Time elapsed 31.619975805282593\n",
            "50/313 --> Time elapsed 39.514856815338135\n",
            "60/313 --> Time elapsed 47.4206440448761\n",
            "70/313 --> Time elapsed 55.299559116363525\n",
            "80/313 --> Time elapsed 63.19548845291138\n",
            "90/313 --> Time elapsed 71.09185028076172\n",
            "100/313 --> Time elapsed 78.975177526474\n",
            "110/313 --> Time elapsed 86.84933543205261\n",
            "120/313 --> Time elapsed 94.74855661392212\n",
            "130/313 --> Time elapsed 102.65275359153748\n",
            "140/313 --> Time elapsed 110.57524490356445\n",
            "150/313 --> Time elapsed 118.48816657066345\n",
            "160/313 --> Time elapsed 126.40192985534668\n",
            "170/313 --> Time elapsed 134.3464024066925\n",
            "180/313 --> Time elapsed 142.2561275959015\n",
            "190/313 --> Time elapsed 150.1831362247467\n",
            "200/313 --> Time elapsed 158.09416913986206\n",
            "210/313 --> Time elapsed 165.98833656311035\n",
            "220/313 --> Time elapsed 173.91573524475098\n",
            "230/313 --> Time elapsed 181.82240414619446\n",
            "240/313 --> Time elapsed 189.75323724746704\n",
            "250/313 --> Time elapsed 197.6648235321045\n",
            "260/313 --> Time elapsed 205.57817316055298\n",
            "270/313 --> Time elapsed 213.4643316268921\n",
            "280/313 --> Time elapsed 221.36062860488892\n",
            "290/313 --> Time elapsed 229.2595431804657\n",
            "300/313 --> Time elapsed 237.15811157226562\n",
            "310/313 --> Time elapsed 245.06759881973267\n",
            "Training took 247.09386324882507\n",
            "Accuracy: 0.9580078125, Time elapsed: 7.97594428062439\n",
            "======== Epoch 3 / 3 ========\n",
            "0/313 --> Time elapsed 0.0016832351684570312\n",
            "10/313 --> Time elapsed 7.882845163345337\n",
            "20/313 --> Time elapsed 15.787106275558472\n",
            "30/313 --> Time elapsed 23.686606407165527\n",
            "40/313 --> Time elapsed 31.59060788154602\n",
            "50/313 --> Time elapsed 39.45950627326965\n",
            "60/313 --> Time elapsed 47.370983362197876\n",
            "70/313 --> Time elapsed 55.27331185340881\n",
            "80/313 --> Time elapsed 63.194299936294556\n",
            "90/313 --> Time elapsed 71.08424305915833\n",
            "100/313 --> Time elapsed 79.01509857177734\n",
            "110/313 --> Time elapsed 86.91275405883789\n",
            "120/313 --> Time elapsed 94.80952882766724\n",
            "130/313 --> Time elapsed 102.70864772796631\n",
            "140/313 --> Time elapsed 110.61611795425415\n",
            "150/313 --> Time elapsed 118.53871583938599\n",
            "160/313 --> Time elapsed 126.44044804573059\n",
            "170/313 --> Time elapsed 134.36755442619324\n",
            "180/313 --> Time elapsed 142.2794373035431\n",
            "190/313 --> Time elapsed 150.1728093624115\n",
            "200/313 --> Time elapsed 158.0693175792694\n",
            "210/313 --> Time elapsed 165.9616391658783\n",
            "220/313 --> Time elapsed 173.8694040775299\n",
            "230/313 --> Time elapsed 181.76215076446533\n",
            "240/313 --> Time elapsed 189.64362502098083\n",
            "250/313 --> Time elapsed 197.52392625808716\n",
            "260/313 --> Time elapsed 205.4286060333252\n",
            "270/313 --> Time elapsed 213.29826402664185\n",
            "280/313 --> Time elapsed 221.20303177833557\n",
            "290/313 --> Time elapsed 229.11598944664001\n",
            "300/313 --> Time elapsed 237.0246386528015\n",
            "310/313 --> Time elapsed 244.91871213912964\n",
            "Training took 246.944509267807\n",
            "Accuracy: 0.9697265625, Time elapsed: 7.997022390365601\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbUm-D33qtpT",
        "outputId": "7fcc5c91-f65f-4fce-c2e4-1cf4205d9043",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "# plot losses\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns \n",
        "\n",
        "plt.plot(losses, 'b-o')\n",
        "plt.title(\"Train Loss\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwU5bX/8c9hX1QcFhMDDItBDRoVHSGJxnBVFjWCcQND3C4GlxBjNHHDGEQxLve6hqhcf+aaBCXGiIIbYiRGE0UGRRC8GERQcGOTJSCb5/fHU5Ppmelhepiurunu7/v16tdU1VPVfSiaOdTz1HPK3B0REZHqmiQdgIiINE5KECIikpYShIiIpKUEISIiaSlBiIhIWkoQIiKSlhKEyC4ys2fM7Oyk4xCJi2kehBQTM9uYstoG2ALsiNbPd/dJOYpjKXCeuz+fi88T2RXNkg5AJJfcfbeK5Z39kjazZu6+PZexiTQ26mISAcysv5ktN7MrzOxj4LdmVmJmT5rZSjNbGy13STnmr2Z2XrR8jpm9bGb/Fe37npkdtwtxtDSzO8zsw+h1h5m1jNo6RjF8ZmZrzOwlM2sStV1hZivMbIOZLTKzY7J0aqSIKUGIVPoy0B7oBowi/Pv4bbReCmwGfr2T4/sBi4COwC3A/zMzq2cMY4BvAIcABwN9gWuitsuA5UAn4EvA1YCb2X7AaOBwd98dGAQsrefnitSgBCFS6Qvgl+6+xd03u/tqd/+zu29y9w3AeOA7Ozl+mbv/j7vvAB4E9ib8Iq+PEcA4d//U3VcC1wFnRm3bovfs5u7b3P0lD4OIO4CWQG8za+7uS9393Xp+rkgNShAilVa6++cVK2bWxszuM7NlZrYe+Buwp5k1reX4jysW3H1TtLhbLfvW5ivAspT1ZdE2gFuBxcBzZrbEzK6MPmsxcAkwFvjUzCab2VcQaSAlCJFK1W/puwzYD+jn7nsAR0Xb69ttVB8fErq0KpRG23D3De5+mbv3BIYAl1aMNbj7Q+5+ZHSsAzfHGKMUCSUIkdrtThh3+MzM2gO/zPL7NzezVimvZsDDwDVm1snMOgLXAn8AMLPvmtlXo3GNdYSupS/MbD8zOzoazP48ivmLLMcqRUgJQqR2dwCtgVXAq8CzWX7/pwm/zCteY4EbgHJgHjAfeD3aBtALeB7YCLwC/MbdZxLGH26K4vwY2Au4KsuxShHSRDkREUlLVxAiIpKWEoSIiKQVa4Iws8HRrM7FFbfkVWu/wMzmm9ncaBZq72h7dzPbHG2fa2b3xhmniIjUFNsYRHSv+DvAAMLsz9nAGe6+MGWfPdx9fbQ8BLjI3QebWXfgSXc/MJbgRESkTnEW6+sLLHb3JQBmNhkYCvw7QVQkh0hbat6HnrGOHTt69+7dd/VwEZGiNGfOnFXu3ildW5wJojPwQcr6ckKtmirM7EfApUAL4OiUph5m9gawHrjG3V/a2Yd1796d8vLyBgctIlJMzGxZbW2JD1K7+wR33we4gsqiZB8Bpe7eh5A8HjKzPaofa2ajzKzczMpXrlyZu6BFRIpAnAliBdA1Zb1LtK02k4GTAKJiaauj5TnAu8C+1Q9w94nuXubuZZ06pb1CEhGRXRRngpgN9DKzHmbWAhgOTE3dwcx6payeAPwz2t6poiCamfUkzCBdEmOsIiJSTWxjEO6+3cxGA9OBpsAD7r7AzMYB5e4+FRhtZscSyhivBSqe73sUMM7MthFqylzg7mviilVERGoqmFIbZWVlrkFqEZH6MbM57l6Wri3xQeqkTZoE3btDkybh56ScPLJeRKTxi/M210Zv0iQYNQo2RY92WbYsrAOMGJFcXCIijUFRX0GMGVOZHCps2hS2i4gUu6JOEO+/X7/tIiLFpKgTRGlp/baLiBSTok4Q48dDmzZVtzVpAmPHJhKOiEijUtQJYsQImDgRunUDM+jQAb74AqZOhe3bk45ORCRZRZ0gICSJpUtDYli1Cu68E6ZMgfPPhwKZIiIiskuK+jbXdC6+GFavhnHjoGNHuPnmpCMSEUmGEkQaY8eGq4lbbgndTpdfnnREIiK5pwSRhhncfTesWQNXXBGSxMiRSUclIpJbShC1aNIEHnwQ1q4Ns6vbt4fvfS/pqEREcqfoB6l3pkUL+POfoW9fGD4cZs5MOiIRkdxRgqhD27bw1FPQqxcMGQIqGCsixUIJIgPt28P06eGupuOOg0WLko5IRCR+ShAZ6twZnnsujE0MGAAffJB0RCIi8VKCqIdevcKVxLp1MHBguBVWRKRQKUHU0yGHwLRpYfb18cfDhg1JRyQiEg8liF1w1FHwyCPw+uvh1tctW5KOSEQk+5QgdtGJJ8IDD8Bf/hLqOe3YkXREIiLZpQTRAGedBbffHuZKXHihivuJSGHRTOoGuuSSMFg9fny4DfbGG5OOSEQkO5QgsuD660OS+NWvQt2myy5LOiIRkYZTgsgCM5gwIRT3+9nPQpI455ykoxIRaRgliCxp2hR+/3v47DM47zwoKYGhQ5OOSkRk12mQOotatoTHHoPDDoNhw+DFF5OOSERk1ylBZNluu8HTT0PPnuFW2DfeSDoiEZFdowQRgw4dQt2mkhIYNAjeeSfpiERE6k8JIiZdusCMGWF54EBYsSLZeERE6ksJIkb77gvPPhvubho4EFavTjoiEZHMKUHE7NBDYepUePddOOEE2Lgx6YhERDITa4Iws8FmtsjMFpvZlWnaLzCz+WY218xeNrPeKW1XRcctMrNBccYZt/794Y9/hNmz4eSTVdxPRPJDbAnCzJoCE4DjgN7AGakJIPKQu3/d3Q8BbgFui47tDQwHDgAGA7+J3i9vDR0K998fxiXOOkvF/USk8YtzolxfYLG7LwEws8nAUGBhxQ7uvj5l/7ZARbm7ocBkd98CvGdmi6P3eyXGeGN37rmVs61LSuCee8IsbBGRxijOBNEZSH0w53KgX/WdzOxHwKVAC+DolGNfrXZs5zTHjgJGAZSWlmYl6Lhddlmo23TTTdCpU6jjJCLSGCU+SO3uE9x9H+AK4Jp6HjvR3cvcvaxTp07xBBiDG28M5ThuuAHuuCPpaERE0ovzCmIF0DVlvUu0rTaTgXt28di8Ygb33hu6m3760zCx7swzk45KRKSqOK8gZgO9zKyHmbUgDDpPTd3BzHqlrJ4A/DNangoMN7OWZtYD6AW8FmOsOde0KTz0EBxzTBibmDYt6YhERKqKLUG4+3ZgNDAdeBt4xN0XmNk4MxsS7TbazBaY2VzCOMTZ0bELgEcIA9rPAj9y94K776dlS5gyBfr0gdNPh5deSjoiEZFK5gXynMyysjIvLy9POoxdsmoVHHkkfPRRqAB7yCFJRyQixcLM5rh7Wbq2xAepJTyqdMYMaNcOBg+GxYuTjkhERAmi0ejaNVSA3bEDBgyADz9MOiIRKXZKEI3I/vvDM8+ELqdBg8JdTiIiSVGCaGTKyuCJJ8IzJL77XfjXv5KOSESKlRJEI3T00fDwwzBrFpx6KmzdmnREIlKMlCAaqZNPhokTw/Mkzj4bvvgi6YhEpNjEOZNaGmjkyPCQoSuuCLOt775bxf1EJHeUIBq5yy8Pg9a33hpuhx07NumIRKRYKEHkgZtvDlcS110XriR+/OOkIxKRYqAEkQfM4L77wm2vF18M7dvDiBFJRyUihU6D1HmiWbNwZ1P//nDOOfD000lHJCKFTgkij7RqFeZIHHRQuP31739POiIRKWRKEHlmjz3CbOuuXcNEunnzko5IRAqVEkQe2muvUNxvt91CSY533006IhEpREoQeaq0NBT327YNBg4MpcJFRLJJCSKPfe1rYbD6k0/ClcTatUlHJCKFRAkiz/XtC48/Dv/3f3DiibBpU9IRiUihUIIoAMceG55v/Y9/wGmnhW4nEZGGUoIoEKeeCvfeG7qczj1Xxf1EpOE0k7qAjBoVSnJcfXWYbX3nnSruJyK7TgmiwFx5ZSjud9tt0KkT/OIXSUckIvlKCaLAmIXKr6tXw7XXhuJ+F12UdFQiko+UIApQkyZw//3httfRo0N30/DhSUclIvlGg9QFqlkzmDwZvv1tOPPM8GQ6EZH6UIIoYK1bw9SpcOCBcMop8MorSUckIvlECaLAtWsXrh6+8hU44QR4662kIxKRfKEEUQS+9KVQ3K9161C36b33ko5IRPKBEkSR6N49FPf7/HMYMCDUbxIR2RkliCJywAFhpvVHH4Xifp99lnREItKYKUEUmW98Ax57DBYuhCFDYPPmpCMSkcYq1gRhZoPNbJGZLTazK9O0X2pmC81snpn9xcy6pbTtMLO50WtqnHEWm0GD4Pe/h5dfhmHDVNxPRNKLLUGYWVNgAnAc0Bs4w8x6V9vtDaDM3Q8CHgVuSWnb7O6HRK8hccVZrIYNgwkTYNo0GDlSxf1EpKY4Z1L3BRa7+xIAM5sMDAUWVuzg7jNT9n8V+EGM8Ug1F14YSnL84hehJMdtt6m4n4hUijNBdAY+SFlfDvTbyf4jgWdS1luZWTmwHbjJ3R+vfoCZjQJGAZSWljY44GI0Zkwo7nfHHaG439VXJx2RiDQWjaIWk5n9ACgDvpOyuZu7rzCznsALZjbf3d9NPc7dJwITAcrKyjxnARcQs3DlsHp1SBYdOsD55ycdlYg0BnEmiBVA15T1LtG2KszsWGAM8B1331Kx3d1XRD+XmNlfgT7Au9WPl4Zr0gQeeCAU97vwwlDc77TTko5KRJIW511Ms4FeZtbDzFoAw4EqdyOZWR/gPmCIu3+asr3EzFpGyx2BI0gZu5Dsa94cHnkEjjgCRowIM69FpLjFliDcfTswGpgOvA084u4LzGycmVXclXQrsBvwp2q3s34NKDezN4GZhDEIJYiYtWkT7mrq3Ru+9z2YNSvpiEQkSeZeGF33ZWVlXl5ennQYBeHjj+HII0OX09/+FmZgi0hhMrM57l6Wrk0zqaWGL385dDG1bBmK+y1dmnREIpIEJQhJq0cPmD4dNm0KSeLTT+s+RkQKixKE1OrrX4ennoLly2HwYFi/PumIRCSXlCBkp771Lfjzn2H+/FDc7/PPk45IRHJFCULqdNxx8LvfhQHr4cNh+/akIxKRXFCCkIyccQbcdRc88QT88IdQIDe/ichONIpSG5IfRo8OJTnGjg0lOW69VcX9RAqZEoTUy7XXhuJ+//3fobjfFVckHZGIxEUJQurFDO68M1xJXHllqNv0wx8mHZWIxEEJQuqtSRP43/8Nz7S+4IKQJE45JemoRCTbNEgtu6RFC3j00fCM6+9/H55/PumIRCTblCBkl7VpA08+CfvtByedBK+9lnREIpJNShDSICUloSTHXnvB8cfD228nHZGIZIsShDTY3nuH4n7NmoW6Te+/n3REIpINShCSFfvsE64kNmwISWLlyqQjEpGGyihBmFlbM2sSLe9rZkPMrHm8oUm+OfjgMCaxbFkoz7FhQ9IRiUhDZHoF8TeglZl1Bp4DzgT+N66gJH8deWS4u2nu3DBwreJ+Ivkr0wRh7r4JOBn4jbufBug5Y5LWCSeEeRIvvBBugVVxP5H8lHGCMLNvAiOAp6JtTeMJSQrBD34QZlxPmRIm06m4n0j+yXQm9SXAVcAUd19gZj2BmfGFJYXg4otD3abrr4eOHeGmm5KOSETqI6ME4e4vAi8CRIPVq9z94jgDk8Jw3XUhSdx8c6gA+/OfJx2RiGQq07uYHjKzPcysLfAWsNDM9E9d6mQGd98Nw4bB5ZfDAw8kHZGIZCrTMYje7r4eOAl4BuhBuJNJpE5Nm4Yn0g0cGCq/TpmSdEQikolME0TzaN7DScBUd98GaNhRMtaiBTz2GPTtGx5bOlMjWCKNXqYJ4j5gKdAW+JuZdQPWxxWUFKa2beGpp6BXLxgyBMrLk45IRHYmowTh7ne5e2d3P96DZcB/xBybFKD27UNJjo4dw2zrRYuSjkhEapPpIHU7M7vNzMqj138TriZE6q1zZ3juufDgoQED4IMPko5IRNLJtIvpAWADcHr0Wg/8Nq6gpPD16gXPPgvr1oXB61Wrko5IRKrLNEHs4+6/dPcl0es6oGecgUnh69MHpk2DpUvDsyRU3E+kcck0QWw2syMrVszsCGBzPCFJMTnqKPjjH+H11+Hkk2HLlqQjEpEKmSaIC4AJZrbUzJYCvwbOr+sgMxtsZovMbLGZXZmm/VIzW2hm88zsL9HdURVtZ5vZP6PX2RnGKXloyJAwge7550MNpx07ko5IRCDzu5jedPeDgYOAg9y9D3D0zo4xs6bABOA4oDdwhpn1rrbbG0CZux8EPArcEh3bHvgl0A/oC/zSzEoy/lNJ3jnrLLjttlAq/KKLVNxPpDGo1xPl3H19NKMa4NI6du8LLI7GLLYCk4Gh1d5vZlRGHOBVoEu0PAiY4e5r3H0tMAMYXJ9YJf/89Kdw9dUwcSJcc03S0YhIptVc07E62jsDqTcwLidcEdRmJKGMR23Hdq4RgNkoYBRAaWlpHeFIPrjhhnBH0403huJ+l9b13xARiU1DEkTWOgHM7AdAGfCdegXgPhGYCFBWVqZOiQJgBr/5DaxZA5ddFpLE2RqBEknEThOEmW0gfSIwoHUd770C6Jqy3iXaVv0zjgXGAN9x9y0px/avduxf6/g8KRBNm8If/hDmSIwcCXvuCUOH1n2ciGTXTscg3H13d98jzWt3d6/r6mM20MvMephZC2A4MDV1BzPrQ6jzNMTdP01pmg4MNLOSaHB6YLRNikTLlqG432GHhVLhL76YdEQixadeg9T14e7bgdGEX+xvA49ET6MbZ2ZDot1uBXYD/mRmc81sanTsGuB6QpKZDYyLtkkR2W03ePpp6NkTTjwR3ngj6YhEiot5gdxPWFZW5uUqD1qQli+HI46AzZvh5Zdh332TjkikcJjZHHcvS9cW2xWESLZ06QIzZoTlgQNhRY2RLBGJgxKE5IV994Vnngl3Nw0cGH6KSLyUICRvHHYYTJ0K774bivtt3Jh0RCKFTQlC8kr//jB5MsyeDaecAlu3Jh2RSOFSgpC8c9JJcP/94aFDZ52l4n4icWnITGqRxJx7LqxeDT//OZSUhNnXVlfxFxGpFyUIyVs/+1mo23TzzdCpE4wbl3REIoVFCULy2q9+FZLE9deHuk0/+UnSEYkUDiUIyWtmcO+9sHYtXHIJtG8PZ56ZdFQihUGD1JL3mjWDhx6CY44JYxPTpiUdkUhhUIKQgtCyJUyZAn36wOmnw0svJR2RSP5TgpCCsfvuYbZ1t27w3e/C3LlJRySS35QgpKB07BjqNrVrB4MHw+LFSUckkr+UIKTgdO0aJtFt3w4DBsCHHyYdkUh+UoKQgrT//qG7adUqGDQo3OUkIvWjBCEF6/DD4fHH4Z13wpjEv/6VdEQi+UUJQgraMcfAww/Dq6/CqaequJ9IfShBSME7+WS47z549lk45xz44oukIxLJD5pJLUXhvPNCcb8rrwwlOe66S8X9ROqiBCFF4/LLw6D1f/1XSBJjxyYdkUjjpgQhRcMMbrklXElcd11IEj/+cdJRiTReShBSVMxg4sRw2+vFF4fifiNGJB2VSOOkQWopOs2ahTub+vcPg9ZPP510RCKNkxKEFKVWreCJJ+Cgg8Ltr3//e9IRiTQ+ShBStPbYI8y27to1TKSbNy/piEQaFyUIKWp77RXqNrVtG0pyLFmSdEQijYcShBS9bt1Ckti6NRT3+/jjpCMSaRyUIESA3r3DYPUnn4Qric8+SzoikeQpQYhE+vULT6V7+2048UTYtCnpiESSpQQhkmLAAJg0KdzVdPrpsG1b0hGJJCfWBGFmg81skZktNrMr07QfZWavm9l2Mzu1WtsOM5sbvabGGadIqtNOg3vugaeegnPPVXE/KV6xzaQ2s6bABGAAsByYbWZT3X1hym7vA+cAP0vzFpvd/ZC44hPZmfPPDyU5xowJs63vvFPF/aT4xFlqoy+w2N2XAJjZZGAo8O8E4e5Lozb9H00anauuCsX9br8dOnWCX/wi6YhEcivOBNEZ+CBlfTnQrx7HtzKzcmA7cJO7P159BzMbBYwCKC0tbUCoIjWZhcqva9bAtdeG4n4XXZR0VCK505iL9XVz9xVm1hN4wczmu/u7qTu4+0RgIkBZWZknEaQUtiZN4P77Q3G/0aNDd9Pw4UlHJZIbcQ5SrwC6pqx3ibZlxN1XRD+XAH8F+mQzOJFMNWsGkyfDt78NZ54ZnkwnUgziTBCzgV5m1sPMWgDDgYzuRjKzEjNrGS13BI4gZexCJNdat4apU+HAA+GUU+CVV5KOSCR+sSUId98OjAamA28Dj7j7AjMbZ2ZDAMzscDNbDpwG3GdmC6LDvwaUm9mbwEzCGIQShCSqXbtw9fCVr8AJJ8BbbyUdkUi8zL0wuu7Lysq8vLw86TCkCCxdCkccAe5hQl2PHklHJLLrzGyOu5ela9NMapF66t4dpk+Hzz+HgQND/SaRQqQEIbILDjwwzLT+8EMYPBjWrUs6IpHsU4IQ2UXf/CY89hgsWABDhsDmzUlHJJJdShAiDTBoEPzud/DSSzBsmIr7SWFRghBpoOHDYcIEmDYNRo5UcT8pHI15JrVI3rjwwlC3qaIkx223qbif5D8lCJEsueaakCTuuCMU97v66qQjEmkYJQiRLDELlV/XrAllwjt0CGXDRfKVxiBEsqhJE3jggTDT+sIL4eKLw7yJJk3Cz0mTko5QJHO6ghDJsubN4ZFHoE8fuPvuyu3LlsGoUWF5xIhkYhOpD11BiMSgTRvYtKnm9k2bQveTSD5QghCJyYpaitsvWxaeTvfkk7ByZW5jEqkPdTGJxKS0NCSD6po3hxtvrJwv0aMH9OtX+erTB1q1ym2sIukoQYjEZPz4MOaQ2tXUpg1MnAgnnQRz5sCsWeH18svhoUQQEsjBB1cmjL59oVevMNAtkksq9y0So0mTwpjD+++HK4rx42sfoP7ww8qEMWsWlJfDxo2hbc89Q6JIvdLo2DF3fw4pXDsr960EIdJI7dgBCxeGZPHaa+HnW29Vdk317Fk1YRxyiLqmpP6UIEQKxMaNVbumZs2qHAxv3jwkidSk8dWvquSH7JwShEgBW7GiZtfUv/4V2kpKanZNdeiQbLzSuChBiBSRHTvCMypSu6YWLKjsmtpnn5pdUy1bJhuzJEcJQqTIbdhQs2vqww9DW4sWNbum9tlHXVPFQglCRGpYvrxm11TFLbnt21ftmurbV11ThWpnCULzIESKVJcu4XXKKWF9+/bQFVXRLTVrFkyfDhX/h/zqV6teZRx8sLqmCp2uIESkVhs2hCuL1CuNjz4KbS1ahFnfqVcZ6prKP+piEpGscK/ZNTVnTmXXVIcONbum2rdPNmbZOXUxiUhWmEHXruF16qlh2/btYQJf6l1Tzz5b2TXVq1fNrqkWLZL7M0jmdAUhIlm3fn3NrqmPPw5tLVtW7Zrq1y8ULFTXVDLUxSQiiXKHDz6o2TW1eXNo79ixZtdUSUmyMRcLdTGJSKLMQrHC0lI47bSwbdu2yq6piu6pZ56p7Jrad9+qVxkHHaSuqVzTFYSINBrr1tXsmvrkk9DWsiUcemjVpNG9u7qmGkpdTCKSl9xDqfTqXVOffx7aO3Wq2TW1557JxpxvEutiMrPBwJ1AU+B+d7+pWvtRwB3AQcBwd380pe1s4Jpo9QZ3fzDOWEWk8TGDbt3C6/TTw7Zt22D+/KpJ46mnKo/Zb7+aXVPNmycTf76L7QrCzJoC7wADgOXAbOAMd1+Ysk93YA/gZ8DUigRhZu2BcqAMcGAOcJi7r63t83QFIVK81q2D2bOrJo1PPw1trVrV7Jrq1k1dUxWSuoLoCyx29yVREJOBocC/E4S7L43avqh27CBghruvidpnAIOBh2OMV0TyVLt2cOyx4QWha2rZsqoJ45574PbbQ/tee1XtlurbN7yHVBVngugMfJCyvhzo14BjO2cpLhEpcGZhALt7dxg2LGzbtg3mzauaNKZNqzxm//2rXmV8/evqmsrr21zNbBQwCqC0tDThaESkMWveHA47LLwuuihsW7u26l1TTz8ND0ajna1b1+yaKi0trq6pOBPECqBrynqXaFumx/avduxfq+/k7hOBiRDGIHYlSBEpXiUlMGBAeEHomlq6tOpVxoQJcNttof1LX6rslurXDw4/vLC7puJMELOBXmbWg/ALfzjw/QyPnQ7caGYVcykHAldlP0QRkUpmoexHjx4wfHjYtnVrza6pqVMr90/XNdUsr/tmKsU6D8LMjifcxtoUeMDdx5vZOKDc3aea2eHAFKAE+Bz42N0PiI79T+Dq6K3Gu/tvd/ZZuotJRHJl7dow8zv12RmrVoW21q1DN1Zq0ujatfF2TWminIhIjNzhvfeqXmW88QZs2RLav/zlml1Te+yRbMwVVItJRCRGZtCzZ3idcUbYtnUrvPlm1aTxxBOV+3/ta1WvMg48sPF1TekKQkQkR9asqTmhb/Xq0NamTc2uqS5d4u+aUheTiEgj5A5LltTsmtq6NbTvvXfVCX2HHw677155/KRJMGZMqFdVWgrjx8OIEfWLQQlCRCRPbNlSs2tq8eLQZga9e4eEAfDQQ5WFCyFchUycWL8koQQhIpLHVq+u2TW1Zk36fbt1C3M5MqVBahGRPNahAwweHF4QuqaaNq18uFKq99/P3uc2yd5biYhILlQ8oS+dbFYdUoIQEclD48eHMYdUbdqE7dmiBCEikodGjAgD0hXPtujWrf4D1HXRGISISJ4aMSK7CaE6XUGIiEhaShAiIpKWEoSIiKSlBCEiImkpQYiISFoFU2rDzFYCyxrwFh2BVVkKJ5sUV/0orvpRXPVTiHF1c/dO6RoKJkE0lJmV11aPJEmKq34UV/0orvoptrjUxSQiImkpQYiISFpKEJUmJh1ALRRX/Siu+lFc9VNUcWkMQkRE0tIVhIiIpKUEISIiaRV8gjCzwWa2yMwWm9mVadpbmtkfo/ZZZtY9pe2qaPsiMxuU47guNbOFZjbPzP5iZt1S2naY2dzoNTXHcZ1jZitTPv+8lLazzeyf0evsHMd1e0pM75jZZyltcZ6vB8zsUzN7q5Z2M7O7orjnmdmhKW1xnq+64lPpxxAAAAZASURBVBoRxTPfzP5hZgentC2Nts81s6w+xzeDuPqb2bqUv69rU9p2+h2IOa6fp8T0VvSdah+1xXm+uprZzOh3wQIz+0mafeL7jrl7wb6ApsC7QE+gBfAm0LvaPhcB90bLw4E/Rsu9o/1bAj2i92maw7j+A2gTLV9YEVe0vjHB83UO8Os0x7YHlkQ/S6LlklzFVW3/HwMPxH2+ovc+CjgUeKuW9uOBZwADvgHMivt8ZRjXtyo+DziuIq5ofSnQMaHz1R94sqHfgWzHVW3fE4EXcnS+9gYOjZZ3B95J828ytu9YoV9B9AUWu/sSd98KTAaGVttnKPBgtPwocIyZWbR9srtvcff3gMXR++UkLnef6e6botVXgS5Z+uwGxbUTg4AZ7r7G3dcCM4DBCcV1BvBwlj57p9z9b0Atj48HQpy/8+BVYE8z25t4z1edcbn7P6LPhdx9vzI5X7VpyHcz23Hl8vv1kbu/Hi1vAN4GOlfbLbbvWKEniM7ABynry6l5cv+9j7tvB9YBHTI8Ns64Uo0k/A+hQiszKzezV83spCzFVJ+4TokuZR81s671PDbOuIi64noAL6Rsjut8ZaK22OM8X/VV/fvlwHNmNsfMRiUQzzfN7E0ze8bMDoi2NYrzZWZtCL9k/5yyOSfny0L3dx9gVrWm2L5jeqJcI2dmPwDKgO+kbO7m7ivMrCfwgpnNd/d3cxTSNOBhd99iZucTrr6OztFnZ2I48Ki770jZluT5atTM7D8ICeLIlM1HRudrL2CGmf1f9D/sXHid8Pe10cyOBx4HeuXoszNxIvB3d0+92oj9fJnZboSkdIm7r8/me+9MoV9BrAC6pqx3ibal3cfMmgHtgNUZHhtnXJjZscAYYIi7b6nY7u4rop9LgL8S/leRk7jcfXVKLPcDh2V6bJxxpRhOtcv/GM9XJmqLPc7zlREzO4jwdzjU3VdXbE85X58CU8he12qd3H29u2+Mlp8GmptZRxrB+Yrs7PsVy/kys+aE5DDJ3R9Ls0t837E4BlYay4twhbSE0OVQMbB1QLV9fkTVQepHouUDqDpIvYTsDVJnElcfwqBcr2rbS4CW0XJH4J9kabAuw7j2Tln+HvCqVw6IvRfFVxItt89VXNF++xMGDC0X5yvlM7pT+6DrCVQdQHwt7vOVYVylhHG1b1Xb3hbYPWX5H8DgHMb15Yq/P8Iv2vejc5fRdyCuuKL2doRxira5Ol/Rn/13wB072Se271jWTm5jfRFG+N8h/LIdE20bR/hfOUAr4E/RP5bXgJ4px46JjlsEHJfjuJ4HPgHmRq+p0fZvAfOjfyDzgZE5jutXwILo82cC+6cc+5/ReVwMnJvLuKL1scBN1Y6L+3w9DHwEbCP08Y4ELgAuiNoNmBDFPR8oy9H5qiuu+4G1Kd+v8mh7z+hcvRn9PY/JcVyjU75fr5KSwNJ9B3IVV7TPOYQbV1KPi/t8HUkY45iX8nd1fK6+Yyq1ISIiaRX6GISIiOwiJQgREUlLCUJERNJSghARkbSUIEREJC0lCJE6VKsGOzeblUTNrHttFURFkqZSGyJ12+zuhyQdhEiu6QpCZBdFzwG4JXoWwGtm9tVoe3cze8Eqn+VRGm3/kplNiQrRvWlm34reqqmZ/U9U7/85M2sd7X+xVT4TZHJCf0wpYkoQInVrXa2LaVhK2zp3/zrwa+COaNvdwIPufhAwCbgr2n4X8KK7H0x49sCCaHsvYIK7HwB8BpwSbb8S6BO9zwVx/eFEaqOZ1CJ1MLON7r5bmu1LgaPdfUlUUO1jd+9gZqsINau2Rds/cveOZrYS6OIphRejEs4z3L1XtH4F0NzdbzCzZ4GNhIqmj3tUxE4kV3QFIdIwXstyfWxJWd5B5djgCYQaO4cCs6NqwyI5owQh0jDDUn6+Ei3/g1AZGGAE8FK0/BfC42Mxs6Zm1q62NzWzJkBXd58JXEGoJFrjKkYkTvofiUjdWpvZ3JT1Z9294lbXEjObR7gKOCPa9mPgt2b2c2AlcG60/SfARDMbSbhSuJBQQTSdpsAfoiRiwF3u/lnW/kQiGdAYhMguisYgytx9VdKxiMRBXUwiIpKWriBERCQtXUGIiEhaShAiIpKWEoSIiKSlBCEiImkpQYiISFr/H8livVUwq5qLAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0T8enD6pV_Qb"
      },
      "source": [
        "# torch.cuda.memory_summary(device=None, abbreviated=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T_7uGtvzWpUr"
      },
      "source": [
        "import os \n",
        "\n",
        "output_dir = './model_save'\n",
        "\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "# tokenizer.save_pretrained(output_dir)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bxTlo72cTanv"
      },
      "source": [
        "def run_test(df_test):\n",
        "\n",
        "    test_encoded_sentences, test_labels = preprocessing(df_test)\n",
        "    test_attention_masks = attention_masks(test_encoded_sentences)\n",
        "\n",
        "    test_inputs = torch.tensor(test_encoded_sentences)\n",
        "    test_labels = torch.tensor(test_labels)\n",
        "    test_masks = torch.tensor(test_attention_masks)\n",
        "\n",
        "    test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "    test_sampler = SequentialSampler(test_data)\n",
        "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "\n",
        "    model.eval()\n",
        "    eval_loss, eval_acc = 0,0\n",
        "    for step, batch in enumerate(test_dataloader):\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        eval_data, eval_masks, eval_labels = batch\n",
        "        with torch.no_grad():\n",
        "            out = model(eval_data,\n",
        "                        token_type_ids = None,\n",
        "                        attention_mask=eval_masks)\n",
        "        logits = out[0]\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        eval_labels = eval_labels.to('cpu').numpy()\n",
        "        batch_acc = compute_accuracy(logits, eval_labels)\n",
        "        eval_acc += batch_acc\n",
        "    print(f\"Accuracy: {eval_acc/(step+1)}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CprT2x6YWDzc",
        "outputId": "efc4083f-f4fc-4464-adc6-b9a01ef08af1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "print(\"Evaluating on english \\n\")\n",
        "df_test = pd.read_csv(\"./english.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on spanish \\n\")\n",
        "df_test = pd.read_csv(\"./spanish.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on french \\n\")\n",
        "df_test = pd.read_csv(\"./french.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on italian \\n\")\n",
        "df_test = pd.read_csv(\"./italian.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on japanese \\n\")\n",
        "df_test = pd.read_csv(\"./japanese.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on russian \\n\")\n",
        "df_test = pd.read_csv(\"./russian.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)\n",
        "\n",
        "print(\"Evaluating on german \\n\")\n",
        "df_test = pd.read_csv(\"./german.test\", delimiter='\\t', header=None, names=['label', 'sentence'])\n",
        "run_test(df_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluating on english \n",
            "\n",
            "Accuracy: 0.965\n",
            "Evaluating on spanish \n",
            "\n",
            "Accuracy: 0.73425\n",
            "Evaluating on french \n",
            "\n",
            "Accuracy: 0.78025\n",
            "Evaluating on italian \n",
            "\n",
            "Accuracy: 0.65675\n",
            "Evaluating on japanese \n",
            "\n",
            "Accuracy: 0.714\n",
            "Evaluating on russian \n",
            "\n",
            "Accuracy: 0.62825\n",
            "Evaluating on german \n",
            "\n",
            "Accuracy: 0.7905\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}