multi_class_text_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multi_class_text_classification.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMubPuhP7C6T+rxd1dXAoYc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cs145442/nlp-projects-with-tf2/blob/master/multi_class_text_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O3Xli4doFWD8",
        "colab_type": "text"
      },
      "source": [
        "### Setting up libraries and dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6arhMdMAzlv_",
        "colab_type": "code",
        "outputId": "121bff4f-1000-40c4-9b5a-80bd44c7b539",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "!pip install bert-for-tf2\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: bert-for-tf2 in /usr/local/lib/python3.6/dist-packages (0.14.4)\n",
            "Requirement already satisfied: params-flow>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from bert-for-tf2) (0.8.2)\n",
            "Requirement already satisfied: py-params>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from bert-for-tf2) (0.9.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (1.18.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (4.41.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (0.1.91)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4r_4EIBcznVX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try:\n",
        "    %tensorflow_version 2.x\n",
        "except Exception:\n",
        "    pass\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "import bert"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wlMOQTm1HQ6a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import pandas as pd\n",
        "import random\n",
        "import math\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "drxmj6Qjz1KL",
        "colab_type": "code",
        "outputId": "78cb3d40-2e90-4e6a-c12b-f4c35db845f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "! ls -l"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 22228\n",
            "drwxr-xr-x 1 root root     4096 May 29 18:19 sample_data\n",
            "-rw-r--r-- 1 root root 22751429 Jun 10 15:31 train.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZDP1WWT8Flwb",
        "colab_type": "text"
      },
      "source": [
        "### Pre Processing the textual data\n",
        "\n",
        "*computation or processing per se, with numbers only. Hence, language modeling.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EPwuKh9Q1d7J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BertTokenizer = bert.bert_tokenization.FullTokenizer\n",
        "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
        "                            trainable=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JBvS25ewGqXm",
        "colab_type": "text"
      },
      "source": [
        "*we're using the bert layer for tokenisation only.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CcFlvEKY2nCQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "to_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = BertTokenizer(vocabulary_file, to_lower_case)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UXWc8dhwHyo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def encode_sentence(sent):\n",
        "    return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sent))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbMZWSfgHFBu",
        "colab_type": "text"
      },
      "source": [
        "*our encoder is ready, let's setup helper functions*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ESgiK97F26KO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess_text(sen):\n",
        "    # Removing html tags\n",
        "    sentence = remove_tags(sen)\n",
        "\n",
        "    # Remove punctuations and numbers\n",
        "    sentence = re.sub('[^a-zA-Z]', ' ', sentence)\n",
        "\n",
        "    # Single character removal\n",
        "    sentence = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', sentence)\n",
        "\n",
        "    # Removing multiple spaces\n",
        "    sentence = re.sub(r'\\s+', ' ', sentence)\n",
        "\n",
        "    return sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IAHN8XBwHKEC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "TAG_RE = re.compile(r'<[^>]+>')\n",
        "\n",
        "def remove_tags(text):\n",
        "    return TAG_RE.sub('', text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "euVUfZRuGbN6",
        "colab_type": "text"
      },
      "source": [
        "*Processing manager is ready! Let's get the data.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E6fa2MsoGUcp",
        "colab_type": "text"
      },
      "source": [
        "### Loading the training data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SCQz6jHbHOCV",
        "colab_type": "code",
        "outputId": "4268e4ad-5149-41fb-ca94-847f4a40d228",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_data = pd.read_csv(\"/content/train.csv\")\n",
        "\n",
        "train_data.isnull().values.any()\n",
        "\n",
        "train_data.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(11314, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8Q4oV2UHfQe",
        "colab_type": "code",
        "outputId": "838b2ba2-758a-4213-9111-29c71a8c933d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_data.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>message</th>\n",
              "      <th>topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>From: lerxst@wam.umd.edu (where's my thing)\\r\\...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>From: guykuo@carson.u.washington.edu (Guy Kuo)...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>From: twillis@ec.ecn.purdue.edu (Thomas E Will...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>From: jgreen@amber (Joe Green)\\r\\nSubject: Re:...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>From: jcm@head-cfa.harvard.edu (Jonathan McDow...</td>\n",
              "      <td>14</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id                                            message  topic\n",
              "0   0  From: lerxst@wam.umd.edu (where's my thing)\\r\\...      7\n",
              "1   1  From: guykuo@carson.u.washington.edu (Guy Kuo)...      4\n",
              "2   2  From: twillis@ec.ecn.purdue.edu (Thomas E Will...      4\n",
              "3   3  From: jgreen@amber (Joe Green)\\r\\nSubject: Re:...      1\n",
              "4   4  From: jcm@head-cfa.harvard.edu (Jonathan McDow...     14"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KtR_ti58H8HX",
        "colab_type": "text"
      },
      "source": [
        "*this is how the raw data looks like, let's pre-process the training data!*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LVXyK_GHkSi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "messages = []\n",
        "message = list(train_data['message'])\n",
        "for msg in message:\n",
        "    messages.append(preprocess_text(msg))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xaa9XVNFIOko",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = train_data['topic']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_nYXeDnNIT_X",
        "colab_type": "code",
        "outputId": "b76a7a20-62c1-4046-8c57-9aeef5ecb159",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.tokenize(\"don't be so judgmental\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['don', \"'\", 't', 'be', 'so', 'judgment', '##al']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EpETZj-KIPbD",
        "colab_type": "text"
      },
      "source": [
        "*some magic happened over there, email me for any queries*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9lMRa-sDIiUS",
        "colab_type": "code",
        "outputId": "9f5d1de0-98ee-49e6-e014-c42d4a3b9c89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.convert_tokens_to_ids(tokenizer.tokenize(\"dont be so judgmental\"))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2123, 2102, 2022, 2061, 8689, 2389]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5C2_7f7-ImMX",
        "colab_type": "text"
      },
      "source": [
        "*Alas! Now we have a representation of textual information in numerical form.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PB75ElRdI8AJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tokenize_message(text_message):\n",
        "    return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text_message))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P_ULH_foJBnZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenized_messages = [tokenize_message(message) for message in messages]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EwNna3fiJrLZ",
        "colab_type": "text"
      },
      "source": [
        "### Preparing data for training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mB6VuDBJOd5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "messages_with_len = [[message, y[i], len(message)]\n",
        "                 for i, message in enumerate(tokenized_messages)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gX7mBjeuJ1_X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "random.shuffle(messages_with_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jnm7xrBwJFzu",
        "colab_type": "text"
      },
      "source": [
        "*shuffling, something we should always do for better tangling*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SH0DEtZGJ8gR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "messages_with_len.sort(key=lambda x: x[2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "72GSGH9VJUM6",
        "colab_type": "text"
      },
      "source": [
        "*to handle the dimension for each sequence model, we pad the sequence as per batch size.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PQS3RVn_KNQS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sorted_messages_labels = [(message_lab[0], message_lab[1]) for message_lab in messages_with_len]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qS85REDyKXau",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "processed_dataset = tf.data.Dataset.from_generator(lambda: sorted_messages_labels, output_types=(tf.int32, tf.int32))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mCT0Pv4QKdki",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "batched_dataset = processed_dataset.padded_batch(BATCH_SIZE, padded_shapes=((None, ), ()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVIaY25RKghx",
        "colab_type": "code",
        "outputId": "9e05d3dc-d48f-4194-dcf3-39816aeba3f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "source": [
        "next(iter(batched_dataset))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(32, 33), dtype=int32, numpy=\n",
              " array([[ 3029,  5334,  2110, ...,     0,     0,     0],\n",
              "        [ 2013,  1044,  2140, ...,     0,     0,     0],\n",
              "        [ 3029,  9502,  2110, ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [ 3029,  2103,  2118, ...,  3521,  4717,     0],\n",
              "        [ 2013, 14722,  3103, ...,  8583,  3212, 23689],\n",
              "        [ 2013, 27859,  5051, ...,  2012, 12864,  3210]], dtype=int32)>,\n",
              " <tf.Tensor: shape=(32,), dtype=int32, numpy=\n",
              " array([18,  1,  8,  4,  4,  7, 17, 10, 10,  6, 15, 10,  6,  4,  3,  3,  7,\n",
              "         7,  8, 15, 10, 15,  1, 14,  4,  9,  7,  6,  1, 18,  5,  2],\n",
              "       dtype=int32)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zSE-IqdDKrWM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "TOTAL_BATCHES = math.ceil(len(sorted_messages_labels) / BATCH_SIZE)\n",
        "TEST_BATCHES = TOTAL_BATCHES // 10\n",
        "batched_dataset.shuffle(TOTAL_BATCHES)\n",
        "test_data = batched_dataset.take(TEST_BATCHES)\n",
        "train_data = batched_dataset.skip(TEST_BATCHES)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8og1pcYILRZ3",
        "colab_type": "text"
      },
      "source": [
        "*keeping the 10% of the batched dataset for evaluation*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_lF6ZqqnLuNa",
        "colab_type": "text"
      },
      "source": [
        "### Setting up the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6AFoY4AK7CS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MCTC_MODEL(tf.keras.Model):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 vocabulary_size,\n",
        "                 embedding_dimensions=128,\n",
        "                 cnn_filters=50,\n",
        "                 dnn_units=512,\n",
        "                 model_output_classes=2,\n",
        "                 dropout_rate=0.1,\n",
        "                 training=False,\n",
        "                 name=\"mctc_model\"):\n",
        "        super(MCTC_MODEL, self).__init__(name=name)\n",
        "        \n",
        "        self.embedding = layers.Embedding(vocabulary_size,\n",
        "                                          embedding_dimensions)\n",
        "        self.cnn_layer1 = layers.Conv1D(filters=cnn_filters,\n",
        "                                        kernel_size=2,\n",
        "                                        padding=\"valid\",\n",
        "                                        activation=\"relu\")\n",
        "        self.cnn_layer2 = layers.Conv1D(filters=cnn_filters,\n",
        "                                        kernel_size=3,\n",
        "                                        padding=\"valid\",\n",
        "                                        activation=\"relu\")\n",
        "        self.cnn_layer3 = layers.Conv1D(filters=cnn_filters,\n",
        "                                        kernel_size=4,\n",
        "                                        padding=\"valid\",\n",
        "                                        activation=\"relu\")\n",
        "        self.pool = layers.GlobalMaxPool1D()\n",
        "        \n",
        "        self.dense_1 = layers.Dense(units=dnn_units, activation=\"relu\")\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        if model_output_classes == 2:\n",
        "            self.last_dense = layers.Dense(units=1,\n",
        "                                           activation=\"sigmoid\")\n",
        "        else:\n",
        "            self.last_dense = layers.Dense(units=model_output_classes,\n",
        "                                           activation=\"softmax\")\n",
        "    \n",
        "    def call(self, inputs, training):\n",
        "        l = self.embedding(inputs)\n",
        "        l_1 = self.cnn_layer1(l) \n",
        "        l_1 = self.pool(l_1) \n",
        "        l_2 = self.cnn_layer2(l) \n",
        "        l_2 = self.pool(l_2)\n",
        "        l_3 = self.cnn_layer3(l)\n",
        "        l_3 = self.pool(l_3) \n",
        "        \n",
        "        concatenated = tf.concat([l_1, l_2, l_3], axis=-1) # (batch_size, 3 * cnn_filters)\n",
        "        concatenated = self.dense_1(concatenated)\n",
        "        concatenated = self.dropout(concatenated, training)\n",
        "        model_output = self.last_dense(concatenated)\n",
        "        \n",
        "        return model_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wSW0EKXwMDxn",
        "colab_type": "text"
      },
      "source": [
        "*our model is ready to invoke. please email me, for any queries!*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KSFhQrOfMsCd",
        "colab_type": "text"
      },
      "source": [
        "*initialising all the hyper parameters*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DVKWUNIyLfGA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VOCAB_LENGTH = len(tokenizer.vocab)\n",
        "EMB_DIM = 200\n",
        "CNN_FILTERS = 100\n",
        "DNN_UNITS = 256\n",
        "OUTPUT_CLASSES = 20\n",
        "\n",
        "DROPOUT_RATE = 0.2\n",
        "\n",
        "NB_EPOCHS = 4"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BSwELFoyM6-d",
        "colab_type": "text"
      },
      "source": [
        "*loading the model*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H70Sp0vbL2pi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_model = MCTC_MODEL(vocabulary_size=VOCAB_LENGTH,\n",
        "                        embedding_dimensions=EMB_DIM,\n",
        "                        cnn_filters=CNN_FILTERS,\n",
        "                        dnn_units=DNN_UNITS,\n",
        "                        model_output_classes=OUTPUT_CLASSES,\n",
        "                        dropout_rate=DROPOUT_RATE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Jf8wp-sM-jT",
        "colab_type": "text"
      },
      "source": [
        "*adding the configuration for loss and optimisation metrics*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sAaAR-jL7_0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if OUTPUT_CLASSES == 2:\n",
        "    text_model.compile(loss=\"binary_crossentropy\",\n",
        "                       optimizer=\"adam\",\n",
        "                       metrics=[\"accuracy\"])\n",
        "else:\n",
        "    text_model.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                       optimizer=\"adam\",\n",
        "                       metrics=[\"sparse_categorical_accuracy\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHFXet5wMDoW",
        "colab_type": "code",
        "outputId": "72f46892-6147-4ed8-90c0-d8e88fc4ac83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "text_model.fit(train_data, epochs=NB_EPOCHS)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/4\n",
            "319/319 [==============================] - 132s 413ms/step - loss: 1.8506 - sparse_categorical_accuracy: 0.4753\n",
            "Epoch 2/4\n",
            "319/319 [==============================] - 131s 412ms/step - loss: 0.4413 - sparse_categorical_accuracy: 0.8748\n",
            "Epoch 3/4\n",
            "319/319 [==============================] - 132s 414ms/step - loss: 0.1066 - sparse_categorical_accuracy: 0.9743\n",
            "Epoch 4/4\n",
            "319/319 [==============================] - 135s 424ms/step - loss: 0.0256 - sparse_categorical_accuracy: 0.9957\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f7383869f98>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iL1pUDn5NMxV",
        "colab_type": "text"
      },
      "source": [
        "*after some hyper parameter tuning we got a training accuracy of 97% at the 3rd epoch*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBnIZ0-rS0L_",
        "colab_type": "code",
        "outputId": "c02a2d41-fa20-4f11-a2f4-1f70e29b4cf1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "results = text_model.evaluate(test_data)\n",
        "print(results)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "35/35 [==============================] - 1s 25ms/step - loss: 0.9605 - sparse_categorical_accuracy: 0.7312\n",
            "[0.960477352142334, 0.731249988079071]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wPrM1sMcNhof",
        "colab_type": "text"
      },
      "source": [
        "*loading the test dataset*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-35Wtr_klVAr",
        "colab_type": "code",
        "outputId": "1d933c4c-8785-496d-f567-ffdfd76b8fe1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "! ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sample_data  test.csv  train.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrTc7h_3puTY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_test_data = pd.read_csv('test.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tU8L92aBp3Bc",
        "colab_type": "code",
        "outputId": "97ce2175-ad77-49c2-beaf-7eb0b57628f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "raw_test_data.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>From: v064mb9k@ubvmsd.cc.buffalo.edu (NEIL B. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>From: Rick Miller &lt;rick@ee.uwm.edu&gt;\\r\\nSubject...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>From: mathew &lt;mathew@mantis.co.uk&gt;\\r\\nSubject:...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>From: bakken@cs.arizona.edu (Dave Bakken)\\r\\nS...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>From: livesey@solntze.wpd.sgi.com (Jon Livesey...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id                                            message\n",
              "0   0  From: v064mb9k@ubvmsd.cc.buffalo.edu (NEIL B. ...\n",
              "1   1  From: Rick Miller <rick@ee.uwm.edu>\\r\\nSubject...\n",
              "2   2  From: mathew <mathew@mantis.co.uk>\\r\\nSubject:...\n",
              "3   3  From: bakken@cs.arizona.edu (Dave Bakken)\\r\\nS...\n",
              "4   4  From: livesey@solntze.wpd.sgi.com (Jon Livesey..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8io_Aw-Fp_dO",
        "colab_type": "code",
        "outputId": "08e11244-5d0e-4fe9-b650-79ab6d153ce5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "raw_test_data.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(7532, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27lEDybdN8cS",
        "colab_type": "text"
      },
      "source": [
        "*writing a simple prediction function*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VJsmGvQAqM9l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_prediction(sentence):\n",
        "    tokens = encode_sentence(sentence)\n",
        "    inputs = tf.expand_dims(tokens, 0)\n",
        "\n",
        "    output = text_model(inputs, training=False)\n",
        "\n",
        "    pred_class = np.argmax(np.array(output))\n",
        "    return pred_class"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GaRmlkSNOSfJ",
        "colab_type": "text"
      },
      "source": [
        "*same textual data pre-processing, pre-processing*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_vqzQdXsW_C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_messages = []\n",
        "test_message = list(raw_test_data['message'])\n",
        "for msg in test_message:\n",
        "    test_messages.append(preprocess_text(msg))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dnr3kKlvvjTN",
        "colab_type": "code",
        "outputId": "feef9c6d-63ba-4578-b40b-211afe3d35b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "get_prediction('How is the topic?')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 147
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "elcD-4QHv_Z1",
        "colab_type": "code",
        "outputId": "ec1e3ce7-260f-4201-cc5f-32680599bf7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "for i in range(0,3):\n",
        "  print(f\"{get_prediction(message[i])}, {y[i]}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7, 7\n",
            "4, 4\n",
            "4, 4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qmkdolN7Oube",
        "colab_type": "text"
      },
      "source": [
        "*validated the labelling, adding the predictions to a csv format file*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znHvlxiZxm7x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_pred_classes = list()\n",
        "for msg in test_messages:\n",
        "  test_pred_classes.append(get_prediction(msg))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nLZ217R68qQ_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission_data = {'id': raw_test_data['id'], 'topic': test_pred_classes}\n",
        "submission_data = pd.DataFrame(submission_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QSC_36qs9OaQ",
        "colab_type": "code",
        "outputId": "99b5b6c5-c6b5-4602-eb02-b712d10964eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "submission_data.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id  topic\n",
              "0   0     14\n",
              "1   1     18\n",
              "2   2      0\n",
              "3   3     18\n",
              "4   4      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 159
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-WMR0aA_9qNc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission_data.to_csv('submission.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3hEaE_ckCTz_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}