Airline_Tweets_Sentiment_Prediction_with_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /Users/a0j01no/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading dataset and applying preprocessing such as:\n",
    "* Cleaning by removing links, special characters, etc\n",
    "* Removing Stop words\n",
    "* Stemming or Lemmatization\n",
    "* Tokenizing, that is converting from text to sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv(\"Tweets-1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = list(tweets.text.values)\n",
    "Y = list(tweets.airline_sentiment.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking if all are marked sentiments\n",
    "len(X) == len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(document, stem=False):\n",
    "    document = document.lower()\n",
    "    \n",
    "    words = word_tokenize(document)\n",
    "    \n",
    "    words = [word for word in words if word not in stopwords.words(\"english\")]\n",
    "    \n",
    "    if stem:\n",
    "        words = [PorterStemmer().stem(word) for word in words]\n",
    "    else:\n",
    "        words = [WordNetLemmatizer().lemmatize(word, pos='v') for word in words]\n",
    "    \n",
    "    document = \" \".join(words)\n",
    "    \n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = []\n",
    "message_set = []\n",
    "for s,l in zip(X,Y):\n",
    "    #print(s)\n",
    "    temp = s\n",
    "    temp = re.sub(r'@[A-Za-z]+', '', temp)                                # removing words with @ signs\n",
    "    temp = re.sub(r'[^\\x00-\\x7F]+', '', re.sub(r\"http\\S+\", \"\", temp))     # removing emoji's\n",
    "    temp = re.sub(r'[^\\w\\s]', '', temp)                                   # removing punctuations\n",
    "    temp = re.sub(r'[0-9]+', '', temp)                                    # removing numbers\n",
    "    filterd_words = [word for word in preprocess(temp).split() if len(word) > 3]\n",
    "    if len(filterd_words) == 0:                             # removing the messages which have no words in it\n",
    "        continue\n",
    "    message_set.append((filterd_words, l))\n",
    "    all_words.extend(filterd_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_features(all_words):\n",
    "    wordlist = nltk.FreqDist(all_words)\n",
    "    word_features = wordlist.keys()\n",
    "    return word_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9664"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(get_word_features(all_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_80 = int(len(message_set)*0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_messages = message_set[:idx_80]\n",
    "test_messages = message_set[idx_80:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_features = get_word_features(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = nltk.apply_features(extract_features, training_messages)\n",
    "testing_set = nltk.apply_features(extract_features, test_messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['plus', 'youve', 'commercials', 'experience', 'tacky'], 'positive')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_messages[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts([\" \".join(words) for (words, sentiment) in training_messages])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tokenizer.texts_to_sequences([\" \".join(words) for (words, sentiment) in training_messages])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pad_sequences(X)\n",
    "Y = pd.get_dummies([sentiment for (words, sentiment) in training_messages])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X) == len(Y)\n",
    "X.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a LSTM model with some dropouts and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, 16, 128)           256000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_9 (Spatial (None, 16, 128)           0         \n",
      "_________________________________________________________________\n",
      "lstm_8 (LSTM)                (None, 196)               254800    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3)                 591       \n",
      "=================================================================\n",
      "Total params: 511,391\n",
      "Trainable params: 511,391\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(2000, 128, input_length=X.shape[1]))\n",
    "model.add(SpatialDropout1D(0.4))\n",
    "model.add(LSTM(196, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.2, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "9302/9302 [==============================] - 26s 3ms/step - loss: 0.6774 - acc: 0.7169\n",
      "Epoch 2/10\n",
      "9302/9302 [==============================] - 23s 2ms/step - loss: 0.5142 - acc: 0.7920\n",
      "Epoch 3/10\n",
      "9302/9302 [==============================] - 22s 2ms/step - loss: 0.4542 - acc: 0.8210\n",
      "Epoch 4/10\n",
      "9302/9302 [==============================] - 23s 2ms/step - loss: 0.4102 - acc: 0.8392\n",
      "Epoch 5/10\n",
      "9302/9302 [==============================] - 23s 3ms/step - loss: 0.3683 - acc: 0.8563\n",
      "Epoch 6/10\n",
      "9302/9302 [==============================] - 22s 2ms/step - loss: 0.3384 - acc: 0.8685\n",
      "Epoch 7/10\n",
      "9302/9302 [==============================] - 22s 2ms/step - loss: 0.3084 - acc: 0.8805\n",
      "Epoch 8/10\n",
      "9302/9302 [==============================] - 22s 2ms/step - loss: 0.2782 - acc: 0.8902\n",
      "Epoch 9/10\n",
      "9302/9302 [==============================] - 24s 3ms/step - loss: 0.2581 - acc: 0.8984\n",
      "Epoch 10/10\n",
      "9302/9302 [==============================] - 22s 2ms/step - loss: 0.2287 - acc: 0.9111\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x160971d90>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=10, batch_size=8, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2326, 3)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2326, 3)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's do comparison of the prediction and actual result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = 0\n",
    "for x in range(0, pred.shape[0]):\n",
    "    temp = Y_test.reset_index(drop=True).iloc[x, :].values\n",
    "    #print(temp)\n",
    "    if list(pred[x]).index(np.max(pred[x])) == list(temp).index(np.max(temp)):\n",
    "        res = res + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7510748065348237"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res/pred.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training accuracy was 93% while Test accuracy is 75% seems like overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}