BagOfWords(TF_IDF).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM3b09BmVsDpYYXejI3/ia5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/riyajaiswal25/NLP/blob/main/BagOfWords(TF_IDF).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "8xNUFIkUVeeN"
      },
      "outputs": [],
      "source": [
        "import numpy as np \n",
        "import pandas as pd "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_imdb = pd.read_csv(\"imdb_labelled.txt\", delimiter='\\t', header=None)\n",
        "data_imdb.columns = [\"Review_text\", \"Review_class\"]\n",
        "\n",
        "data_amazon = pd.read_csv(\"amazon_cells_labelled.txt\", delimiter='\\t', header=None)\n",
        "data_amazon.columns = [\"Review_text\", \"Review_class\"]\n",
        "\n",
        "data_yelp = pd.read_csv(\"yelp_labelled.txt\", delimiter='\\t', header=None)\n",
        "data_yelp.columns = [\"Review_text\", \"Review_class\"]\n",
        "\n",
        "data = pd.concat([data_imdb, data_amazon, data_yelp])\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "cfsl9hwAV8K7",
        "outputId": "253937e8-3561-47e8-fc61-73d838f8a00f"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           Review_text  Review_class\n",
              "0    A very, very, very slow-moving, aimless movie ...             0\n",
              "1    Not sure who was more lost - the flat characte...             0\n",
              "2    Attempting artiness with black & white and cle...             0\n",
              "3         Very little music or anything to speak of.               0\n",
              "4    The best scene in the movie was when Gerardo i...             1\n",
              "..                                                 ...           ...\n",
              "995  I think food should have flavor and texture an...             0\n",
              "996                           Appetite instantly gone.             0\n",
              "997  Overall I was not impressed and would not go b...             0\n",
              "998  The whole experience was underwhelming, and I ...             0\n",
              "999  Then, as if I hadn't wasted enough of my life ...             0\n",
              "\n",
              "[2748 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8673c15e-9c75-406d-b313-d6364fe9dd9a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Review_text</th>\n",
              "      <th>Review_class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A very, very, very slow-moving, aimless movie ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Not sure who was more lost - the flat characte...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Attempting artiness with black &amp; white and cle...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Very little music or anything to speak of.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The best scene in the movie was when Gerardo i...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>I think food should have flavor and texture an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>Appetite instantly gone.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>Overall I was not impressed and would not go b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>The whole experience was underwhelming, and I ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>Then, as if I hadn't wasted enough of my life ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2748 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8673c15e-9c75-406d-b313-d6364fe9dd9a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8673c15e-9c75-406d-b313-d6364fe9dd9a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8673c15e-9c75-406d-b313-d6364fe9dd9a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import nltk\n",
        "import string\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer"
      ],
      "metadata": {
        "id": "Ja-FmNuDWC90"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('corpus')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N8c4hlTrWQ2V",
        "outputId": "61c77f46-9117-413f-a154-aacb5ad4c6e3"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Error loading corpus: Package 'corpus' not found in index\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_text(df):\n",
        "    all_reviews = list()\n",
        "    lines = df[\"Review_text\"].values.tolist()\n",
        "    for text in lines:\n",
        "        text = text.lower()\n",
        "        pattern = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
        "        text = pattern.sub('', text)\n",
        "        text = re.sub(r\"[,.\\\"!@#$%^&*(){}?/;`~:<>+=-]\", \"\", text)\n",
        "        tokens = word_tokenize(text)\n",
        "        table = str.maketrans('', '', string.punctuation)\n",
        "        stripped = [w.translate(table) for w in tokens]\n",
        "        words = [word for word in stripped if word.isalpha()]\n",
        "        stop_words = set(stopwords.words(\"english\"))\n",
        "        stop_words.discard(\"not\")\n",
        "        PS = PorterStemmer()\n",
        "#         words = [w for w in words if not w in stop_words]\n",
        "        words = [PS.stem(w) for w in words if not w in stop_words]\n",
        "        words = ' '.join(words)\n",
        "        all_reviews.append(words)\n",
        "    return all_reviews\n",
        "\n",
        "all_reviews = clean_text(data)\n",
        "all_reviews[0:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vx4VwyelWFEy",
        "outputId": "7b387e32-d0be-4235-99a6-9d731a70c9ba"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['slowmov aimless movi distress drift young man',\n",
              " 'not sure lost flat charact audienc nearli half walk',\n",
              " 'attempt arti black white clever camera angl movi disappoint becam even ridicul act poor plot line almost nonexist',\n",
              " 'littl music anyth speak',\n",
              " 'best scene movi gerardo tri find song keep run head',\n",
              " 'rest movi lack art charm mean empti work guess empti',\n",
              " 'wast two hour',\n",
              " 'saw movi today thought good effort good messag kid',\n",
              " 'bit predict',\n",
              " 'love cast jimmi buffet scienc teacher',\n",
              " 'babi owl ador',\n",
              " 'movi show lot florida best made look appeal',\n",
              " 'song best muppet hilari',\n",
              " 'cool',\n",
              " 'right case movi deliv everyth almost right face',\n",
              " 'averag act main person low budget clearli see',\n",
              " 'review long overdu sinc consid tale two sister singl greatest film ever made',\n",
              " 'put gem movi term screenplay cinematographi act postproduct edit direct aspect filmmak',\n",
              " 'practic perfect true masterpiec sea faux masterpiec',\n",
              " 'structur film easili tightli construct histori cinema think film someth vital import occur everi minut word content level film enough easili fill dozen film anyon right mind ask anyth movi quit simpli highest superl form cinema imagin ye film requir rather signific amount puzzlesolv piec fit togeth creat beauti pictur short film certainli pull punch graphic far best part game number one best th game seri deserv strong love insan game massiv level massiv unlock charact massiv game wast money game kind money wast properli actual graphic good time today graphic crap say canada fun game aye game rock buy play enjoy love pure brillianc flick doom concept idea lame take minor charact mediocr film make complet nonsequel chang tone pgrate famili movi nt least bit interest not confirm film would unfunni gener also manag give away entir movi not exagger everi moment everi plot point everi joke told trailer not funni even talent carrel ca nt save costar nt fare much better peopl like morgan freeman jonah hill ed helm wast stori predict lazi real effect work presenc anim integr scene worst obviou bluegreenscreen work ever seen whatev cost much nt translat qualiti sure film succe despit perhap obvious meagr budget glad film nt go obviou choic lesser film certainli would addit one love song ever written french cancan also boast one cutest lead ladi ever grace screen hard not fall headoverheel love girl neg insipid enough caus regret anoth hour life wast front screen long whini pointless recommend wait futur effort let one go excel cast stori line perform total believ ann hech utterli convinc sam shepard portray gung ho marin sober sat rivet tv screen give one resound think tom hank good actor enjoy read book children littl disappoint movi one charact total annoy voic give feel fingernail chalkboard total unnecessari trainrol coaster scene absolut warmth charm scene charact movi total grate nerv perform not improv improvis actor twice much worri not whether deliv line well whether line good quit honestli often not good often dialogu nt realli follow one line anoth fit surround crackl unpredict youth energi honestli found hard follow concentr meander badli gener great thing would nt say worth hour time though suspens builder good cross line g pg especi like nonclich choic parent movi could predict dialog verbatim write movi made better select want movi not gross give chill great choic alexand nevski great film amaz film artist one import whoever live glad pretenti piec nt plan dodg stratu big shot gon na help movi maker nt restrain movi busi québec']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "TV = TfidfVectorizer(min_df=3)\n",
        "X = TV.fit_transform(all_reviews).toarray()\n",
        "y = data[\"Review_class\"].to_numpy()\n",
        "print(np.shape(X))\n",
        "print(np.shape(y))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xFjyAFbvWcGo",
        "outputId": "3a6a2d7d-1755-4efa-e55e-5325b1e87f2c"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(2748, 1192)\n",
            "(2748,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qD6Hvj4HXOeS",
        "outputId": "891dc137-d530-4fe2-d2b1-06aed7df4a40"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.        , 0.        , 0.        , ..., 0.69902343, 0.        ,\n",
              "       0.        ])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)\n",
        "\n",
        "# from sklearn.tree import DecisionTreeClassifier\n",
        "# model = DecisionTreeClassifier(criterion=\"entropy\", random_state=41)\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "model = GaussianNB()\n",
        "model.fit(X_train,y_train)\n",
        "\n",
        "y_pred = model.predict(X_test)\n",
        "\n",
        "from sklearn.metrics import accuracy_score, f1_score, precision_score\n",
        "print(accuracy_score(y_test, y_pred))\n",
        "print(f1_score(y_test, y_pred))\n",
        "print(precision_score(y_test, y_pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jzkl7EGxXXKw",
        "outputId": "788ab2ec-a70b-45a6-bfed-8c778a4b9caf"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.68\n",
            "0.6986301369863013\n",
            "0.6666666666666666\n"
          ]
        }
      ]
    }
  ]
}