Facial_Emotion_Recognition.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Facial Emotion Recognition.ipynb ",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShivamSinghal1/FacialEmotionRecognition/blob/master/Facial_Emotion_Recognition.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9QEGLAovABPW"
      },
      "source": [
        "### Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JFPkk9LUJPzV"
      },
      "source": [
        "import math\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import time\n",
        "\n",
        "import seaborn as sns\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27N0euP5Ad7M"
      },
      "source": [
        "### Importing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v2n1p2cbZgEo"
      },
      "source": [
        "df = pd.read_csv('/content/drive/My Drive/Dataset/fer2013.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhP-fD5Feo9u"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cKnmzTZYesWH"
      },
      "source": [
        "emotion_label_to_text = {0:'anger', 1:'disgust', 2:'fear', 3:'happiness', 4: 'sadness', 5: 'surprise', 6: 'neutral'}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "APpLtj2K_Kuf"
      },
      "source": [
        "df.emotion.value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lEugVs1D_6dQ"
      },
      "source": [
        "### Visualize Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_hgeksZALgO"
      },
      "source": [
        "fig = plt.figure(1, (14, 14))\n",
        "\n",
        "k = 0\n",
        "for label in sorted(df.emotion.unique()):\n",
        "    for j in range(1):\n",
        "        px = df[df.emotion==label].pixels.iloc[k]\n",
        "        px = np.array(px.split(' ')).reshape(48, 48).astype('float32')\n",
        "\n",
        "        k += 1\n",
        "        ax = plt.subplot(7, 7, k)\n",
        "        ax.imshow(px , cmap='gray')\n",
        "        ax.set_xticks([])\n",
        "        ax.set_yticks([])\n",
        "        ax.set_title(emotion_label_to_text[label])\n",
        "        plt.tight_layout()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-UZum1Kj6_M7"
      },
      "source": [
        "### Input and Output Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L_m-E9B47qKE"
      },
      "source": [
        "X = df.pixels.apply(lambda x: np.array(x.split(' ')).astype('float32'))\n",
        "X = np.stack(X, axis=0)\n",
        "Y = np.array(df['emotion'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l2PZ9QFU71lF"
      },
      "source": [
        "X.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NRwjY9Gc73E0"
      },
      "source": [
        "Y.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Bkn66Lr3AJC"
      },
      "source": [
        "# Preprocessing\n",
        "Face Alignment "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XBPMzNSS2vd3"
      },
      "source": [
        "import cv2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WKe-bgr41DhI"
      },
      "source": [
        "face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')\n",
        "eye_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RauEPNDt1UNq"
      },
      "source": [
        "def face_align(gray):\n",
        "  sz = gray.shape\n",
        "  # Creating variable eyes\n",
        "  eyes = eye_cascade.detectMultiScale(gray, 1.3, 5)\n",
        "  if len(eyes) > 1 :\n",
        "    index=0\n",
        "    # Creating for loop in order to divide one eye from another\n",
        "    for (ex , ey,  ew,  eh) in eyes:\n",
        "      if index == 0:\n",
        "        eye_1 = (ex, ey, ew, eh)\n",
        "      elif index == 1:\n",
        "        eye_2 = (ex, ey, ew, eh)\n",
        "      index += 1\n",
        "    if eye_1[0] < eye_2[0]:\n",
        "      left_eye = eye_1\n",
        "      right_eye = eye_2\n",
        "    else:\n",
        "      left_eye = eye_2\n",
        "      right_eye = eye_1\n",
        "        \n",
        "    # Calculating coordinates of a central points of the rectangles\n",
        "    left_eye_center = (int(left_eye[0] + (left_eye[2] / 2)), int(left_eye[1] + (left_eye[3] / 2)))\n",
        "    left_eye_x = left_eye_center[0] \n",
        "    left_eye_y = left_eye_center[1]\n",
        "        \n",
        "    right_eye_center = (int(right_eye[0] + (right_eye[2]/2)), int(right_eye[1] + (right_eye[3]/2)))\n",
        "    right_eye_x = right_eye_center[0]\n",
        "    right_eye_y = right_eye_center[1]\n",
        "\n",
        "    if left_eye_y > right_eye_y:\n",
        "      A = (right_eye_x, left_eye_y)\n",
        "      # Integer -1 indicates that the image will rotate in the clockwise direction\n",
        "      direction = -1 \n",
        "    else:\n",
        "      A = (left_eye_x, right_eye_y)\n",
        "      # Integer 1 indicates that image will rotate in the counter clockwise direction\n",
        "      direction = 1 \n",
        "        \n",
        "    delta_x = right_eye_x - left_eye_x\n",
        "    delta_y = right_eye_y - left_eye_y\n",
        "    angle=np.arctan(delta_y/delta_x)\n",
        "    angle = (angle * 180) / np.pi\n",
        "\n",
        "    # Width and height of the image\n",
        "    h, w = gray.shape[:2]\n",
        "    # Calculating a center point of the image\n",
        "    # Integer division \"//\"\" ensures that we receive whole numbers\n",
        "    center = (w // 2, h // 2)\n",
        "    # Defining a matrix M and calling\n",
        "    # cv2.getRotationMatrix2D method\n",
        "    M = cv2.getRotationMatrix2D(center, (angle), 1.0)\n",
        "    # Applying the rotation to our image using the\n",
        "    # cv2.warpAffine method\n",
        "    gray = cv2.warpAffine(gray, M, (w, h))\n",
        "\n",
        "    #Again detecting face\n",
        "    faces = face_cascade.detectMultiScale(gray, 1.3, 5)\n",
        "    if len(faces) > 0 : \n",
        "      (x,y,w,h) = faces[0]\n",
        "      gray = gray[y:y+h, x:x+w]\n",
        "\n",
        "  gray = cv2.resize(gray,sz) \n",
        "  return gray"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9A5NId4E4hXB"
      },
      "source": [
        "X_new = []\n",
        "for j in range(X.shape[0]):\n",
        "  px = np.array(X[j]).reshape(48,48)\n",
        "  px = np.array(px, dtype='uint8')\n",
        "  px = face_align(px)\n",
        "  X_new.append(np.array(px).reshape(48*48))\n",
        "X = np.array(X_new)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0cYBbal5TotN"
      },
      "source": [
        "px = cv2.imread('/content/drive/My Drive/Dataset/Robert-Downey-Jr-Avengers-Endgame.jpg')\n",
        "plt.imshow(px)\n",
        "gray=cv2.cvtColor(px, cv2.COLOR_BGR2GRAY)\n",
        "gray = face_align(gray)\n",
        "plt.imshow(gray)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fU-qWQvU1gEK"
      },
      "source": [
        "### Confusion Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8w2jIC41is1"
      },
      "source": [
        "# Plot confusion matrix \n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "\n",
        "def create_confmat(true_labels, predicted_labels, columns, colour = 'Greens', size = (20,14)):\n",
        "    \n",
        "    cm = confusion_matrix(true_labels, predicted_labels) \n",
        "    cm_df = pd.DataFrame(cm,\n",
        "    index = [col for col in columns], \n",
        "    columns = [col for col in columns])\n",
        "    plt.figure(figsize=(18,16))\n",
        "    sns.heatmap(cm_df, annot = True, cmap = colour, fmt='g', linewidths=.2)\n",
        "    plt.title('Confusion Matrix', fontsize = 20)\n",
        "    plt.ylabel('True label', fontsize = 18)\n",
        "    plt.xlabel('Predicted label', fontsize = 18)\n",
        "    plt.tick_params(axis='both', labelsize=14)\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u2L5tZPzIjmo"
      },
      "source": [
        "### Train and Test Split for KNN\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0c1CFeRC-Snf"
      },
      "source": [
        "X_train, X_valid, y_train, y_valid = train_test_split(X, Y,\n",
        "                                                    shuffle=True, stratify=Y,\n",
        "                                                    test_size=0.01, random_state=2)\n",
        "X_train.shape, X_valid.shape, y_train.shape, y_valid.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RQub0P71cFS"
      },
      "source": [
        "## SVM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RevNX7LUNRYk"
      },
      "source": [
        "from sklearn.svm import SVC\n",
        "model = SVC()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v7t_E4tPIGiL"
      },
      "source": [
        "model.fit(X_train,y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8kW9bVIIOfk"
      },
      "source": [
        "model.score(X_valid , y_valid)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0lXJt_CiBQ_D"
      },
      "source": [
        "We have now X_train (input) and y_train (output) Data for Training and X_valid and y_valid for Testing."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ma2HJ4hSQi_u"
      },
      "source": [
        "# KNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zB7yTAp8nR6I"
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "scaler = StandardScaler()\n",
        "scaler.fit(X_train)\n",
        "\n",
        "X_train = scaler.transform(X_train)\n",
        "X_valid = scaler.transform(X_valid)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pcdJ81kcgtu3"
      },
      "source": [
        "def distance(v1, v2):\n",
        "\t# Eucledian \n",
        "\treturn np.sqrt(((v1-v2)**2).sum())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBcUOaifAUJV"
      },
      "source": [
        "def knn(X_train, Y_train , test , num ):\n",
        "\tdist = []\n",
        "\t\n",
        "\tfor i in range(X_train.shape[0]):\n",
        "\t\t# Compute the distance from test point\n",
        "\t\td = distance(test, X_train[i])\n",
        "\t\tdist.append([d, Y_train[i]])\n",
        "\t# Sort based on distance \n",
        "\tdk = sorted(dist, key=lambda x : x[0])\n",
        "\t# Retrieve only the labels\n",
        "\tlabels = np.array(dk)[:,-1]\n",
        "\toutput = np.zeros(7)\n",
        "\tanswer = []\n",
        "\tj = 0\n",
        "\tfor k in range(num):\n",
        "\t\toutput[int(labels[k])]+=1\n",
        "\t\t#u,indices = np.unique(output,return_counts = True)\n",
        "\t\t#if indices[-1] == 1 or k == num - 1: \n",
        "\t\t#\twhile j <= k : \n",
        "\t\tanswer.append( np.argmax(output) )\n",
        "\t\t#\t\tj+=1\n",
        "\treturn answer\n",
        "\n",
        "\t# Get frequencies of each label\n",
        "\t#output = np.unique(labels, return_counts=True)\n",
        "\t# Find max frequency and corresponding label\n",
        "\t#index = np.argmax(output[1])\n",
        "\t#return output[0][index]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULNwJs0cYOYG"
      },
      "source": [
        "def wknn(X_train, Y_train , test , num ):\n",
        "\tdist = []\n",
        "\tfor i in range(X_train.shape[0]):\n",
        "\t\t# Compute the distance from test point\n",
        "\t\td = distance(test, X_train[i])\n",
        "\t\tif d == 0:\n",
        "\t\t\td = 0.00000000001 \n",
        "\t\tdist.append([1/(d*d), Y_train[i]])\n",
        "\t# Sort reverse based on distance and get top k\n",
        "\tdk = sorted(dist, key=lambda x : x[0] , reverse = True)\n",
        "\toutput = np.zeros(7)\n",
        "\tanswer = []\n",
        "\tfor k in range(num):\n",
        "\t\toutput[int(dk[k][1])] += dk[k][0] \n",
        "\t\tanswer.append( np.argmax(output) )\n",
        "\treturn answer\n",
        "\n",
        "\t# Get frequencies of each label\n",
        "\t#output = np.unique(labels, return_counts=True)\n",
        "\t# Find max frequency and corresponding label\n",
        "\t#index = np.argmax(output[1])\n",
        "\t#return output[0][index]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U8O5wPSmCP7k"
      },
      "source": [
        "start_time = time.time()\n",
        "\n",
        "num = 10\n",
        "cnt = np.zeros(num)\n",
        "y_pred_manually = []\n",
        "\n",
        "for i in range(X_valid.shape[0]):\n",
        "  answer = knn(X_train , y_train , X_valid[i] ,num)\n",
        "  y_pred_manually.append(answer[0])\n",
        "  for j in range(num):\n",
        "    cnt[j] += answer[j] == y_valid[i]\n",
        "\n",
        "accuracy = cnt/X_valid.shape[0]\n",
        "\n",
        "print(\"Maximum Accuracy :\",max(accuracy)*100 ,\"%\")\n",
        "print(\"Value of K :\",np.argmax(accuracy)+1)\n",
        "print(\"Time taken to execute : %s seconds \" % (time.time() - start_time))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yMAfyiVof-S_"
      },
      "source": [
        "plt.style.use('seaborn')\n",
        "\n",
        "plt.plot(list(range(1,11)) , accuracy , marker= 'o')\n",
        "plt.legend(loc = 'upper left',fontsize = 16)\n",
        "plt.xlabel(\"Value of K\",fontsize = 20)\n",
        "plt.ylabel(\"Accuracy\",fontsize = 20)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UioV7wfgRcXR"
      },
      "source": [
        "start_time = time.time()\n",
        "\n",
        "num = 10\n",
        "cnt = np.zeros(num)\n",
        "y_pred_manually = []\n",
        "\n",
        "for i in range(X_valid.shape[0]):\n",
        "  answer = wknn(X_train , y_train , X_valid[i] ,num)\n",
        "  y_pred_manually.append(answer[0])\n",
        "  for j in range(num):\n",
        "    cnt[j] += answer[j] == y_valid[i]\n",
        "\n",
        "accuracy = cnt/X_valid.shape[0]\n",
        "\n",
        "print(\"Maximum Accuracy :\",max(accuracy)*100 ,\"%\")\n",
        "print(\"Value of K :\",np.argmax(accuracy)+1)\n",
        "print(\"Time taken to execute : %s seconds \" % (time.time() - start_time))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euxRCyB3Re7v"
      },
      "source": [
        "plt.style.use('seaborn')\n",
        "\n",
        "plt.plot(list(range(1,11)) , accuracy , marker= 'o')\n",
        "plt.legend(loc = 'upper left',fontsize = 16)\n",
        "plt.xlabel(\"Value of K\",fontsize = 20)\n",
        "plt.ylabel(\"Accuracy\",fontsize = 20)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7FDqjk2KQdym"
      },
      "source": [
        "Applying KNN using SkLearn Library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QGzu5duARglA"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "classifier = KNeighborsClassifier(n_neighbors=1)\n",
        "classifier.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJS6uh7MSqfG"
      },
      "source": [
        "start_time = time.time()\n",
        "y_pred = classifier.predict(X_valid)\n",
        "print(\"Time Taken : %s seconds \" % (time.time() - start_time))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQdqpwksStFp"
      },
      "source": [
        "cnt = 0\n",
        "for i in range(X_valid.shape[0]):\n",
        "   cnt += y_pred[i] == y_valid[i]\n",
        "print(\"Accuracy :\", cnt/X_valid.shape[0]*100, \"%\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nxxlnMpku5Kj"
      },
      "source": [
        "Accuracy with Sklearn Library function is exactly same as manually implemented code."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6evJgz-0vE1T"
      },
      "source": [
        "# Checking manually code prediction with the library code prediction\n",
        "cnt = 0\n",
        "for i in range(X_valid.shape[0]):\n",
        "   cnt += y_pred[i] == y_pred_manually[i]\n",
        "print(\"Accuracy :\", cnt/X_valid.shape[0]*100, \"%\")\n",
        "# All Prediction on these examples are exactly same"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWa9WZat2oEz"
      },
      "source": [
        "END KNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XRWqTCIRWhrJ"
      },
      "source": [
        "# Applying CNN \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D80uazEFeXZf"
      },
      "source": [
        "X_train, X_valid, y_train, y_valid = train_test_split(X, Y,\n",
        "                                                    shuffle=True, stratify=Y,\n",
        "                                                    test_size=0.1, random_state=2)\n",
        "X_train.shape, X_valid.shape, y_train.shape, y_valid.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z9csGCr1hyJt"
      },
      "source": [
        "import keras\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.models import Sequential , load_model\n",
        "from keras.layers import Convolution2D, Activation, BatchNormalization, MaxPooling2D, Dropout, Dense, Flatten, AveragePooling2D\n",
        "from keras.initializers import  RandomNormal\n",
        "from keras.layers import Conv2D,MaxPooling2D\n",
        "from keras.optimizers import RMSprop,SGD,Adam\n",
        "from keras.callbacks import ModelCheckpoint, TensorBoard ,EarlyStopping, ReduceLROnPlateau"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IMYY6LAMMIdv"
      },
      "source": [
        "from sklearn import preprocessing\n",
        "X_train = preprocessing.normalize(X_train)\n",
        "X_valid = preprocessing.normalize(X_valid)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "acHuu7v0VbU1"
      },
      "source": [
        "X_train = X_train.reshape(X_train.shape[0],48,48,1)\n",
        "X_valid = X_valid.reshape(X_valid.shape[0],48,48,1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nKyWBvOuPrR0"
      },
      "source": [
        "from keras.utils import np_utils\n",
        "num_classes = 7\n",
        "print(\"Shape before one-hot encoding: \", y_train.shape)\n",
        "y_train = np_utils.to_categorical(y_train, num_classes)\n",
        "y_valid = np_utils.to_categorical(y_valid, num_classes)\n",
        "print(\"Shape after one-hot encoding: \", y_train.shape)\n",
        "\n",
        "print(X_train.shape , y_train.shape)\n",
        "print(X_valid.shape , y_valid.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yVLpYsAqQeGj"
      },
      "source": [
        "def get_cnn_model():\n",
        "    model = Sequential()\n",
        "\n",
        "    model.add(Convolution2D(64, (3, 1), padding='same', input_shape=(48,48,1)))\n",
        "    model.add(Convolution2D(64, (1, 3), padding='same'))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Convolution2D(128, (3, 1), padding='same'))\n",
        "    model.add(Convolution2D(128, (1, 3), padding='same'))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Convolution2D(256, (3, 1), padding='same'))\n",
        "    model.add(Convolution2D(256, (1, 3), padding='same'))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Convolution2D(512, (3, 1), padding='same'))\n",
        "    model.add(Convolution2D(512, (1, 3), padding='same'))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='same'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Flatten())\n",
        "\n",
        "    model.add(Dense(512))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Dense(256))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Dropout(0.25))\n",
        "\n",
        "    model.add(Dense(7))\n",
        "    model.add(Activation('softmax'))\n",
        "    return model\n",
        "\n",
        "model = get_cnn_model()\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8uNyf8tQj6O"
      },
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# Saving model each time it achieves lower loss on the validation set\n",
        "filepath='/content/drive/My Drive/Dataset/VGG16.hdf5'\n",
        "checkpointer = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n",
        "\n",
        "earlystop = EarlyStopping(monitor='val_loss',\n",
        "                          min_delta=0,\n",
        "                          patience=8,\n",
        "                          verbose=1,\n",
        "                          restore_best_weights=True, mode = 'auto'\n",
        "                          )\n",
        "\n",
        "reduce_lr = ReduceLROnPlateau(monitor='val_loss',\n",
        "                              factor=0.9,\n",
        "                              patience=3,\n",
        "                              verbose=1,\n",
        "                              min_delta=0.0001)\n",
        "\n",
        "\n",
        "history = model.fit(X_train , y_train, batch_size = 32\n",
        "                   , epochs=35, \n",
        "                    validation_data=(X_valid, y_valid),\n",
        "                    callbacks=[checkpointer,earlystop,reduce_lr]\n",
        "                             )\n",
        "\n",
        "pd.DataFrame(history.history).to_csv(\"history.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxEp2ViKU4O7"
      },
      "source": [
        "fig = plt.figure()\n",
        "plt.subplot(2,1,1)\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='lower right')\n",
        "\n",
        "plt.subplot(2,1,2)\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper right')\n",
        "\n",
        "plt.tight_layout()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtxQkAKbV_sw"
      },
      "source": [
        "# Plot Confusion Matrix\n",
        "columns = {'anger', 'disgust', 'fear', 'happiness', 'sadness', 'surprise', 'neutral'}\n",
        "cm_df = pd.DataFrame(cm,\n",
        "    index = [col for col in columns], \n",
        "    columns = [col for col in columns])\n",
        "\n",
        "plt.figure(figsize=(18,16))\n",
        "sns.heatmap(cm_df, annot = True, cmap = 'Greens', fmt='g', linewidths=.2)\n",
        "plt.title('Confusion Matrix', fontsize = 20)\n",
        "plt.ylabel('True label', fontsize = 18)\n",
        "plt.xlabel('Predicted label', fontsize = 18)\n",
        "plt.tick_params(axis='both', labelsize=14)\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "98LIJLM5YaXi"
      },
      "source": [
        "## END OF CNN "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M9kDCprAmwwv"
      },
      "source": [
        "def precision(label, confusion_matrix):\n",
        "    col = confusion_matrix[:, label]\n",
        "    return confusion_matrix[label, label] / col.sum()\n",
        "    \n",
        "def recall(label, confusion_matrix):\n",
        "    row = confusion_matrix[label, :]\n",
        "    return confusion_matrix[label, label] / row.sum()\n",
        "  \n",
        "def f1score(label , confusion_matrix):\n",
        "    p = precision(label , confusion_matrix)\n",
        "    r = recall(label , confusion_matrix)\n",
        "    return 2*p*r/(p+r)\n",
        "\n",
        "def precision_macro_average(confusion_matrix):\n",
        "    rows, columns = confusion_matrix.shape\n",
        "    sum_of_precisions = 0\n",
        "    for label in range(rows):\n",
        "        sum_of_precisions += precision(label, confusion_matrix)\n",
        "    return sum_of_precisions / rows\n",
        "\n",
        "def recall_macro_average(confusion_matrix):\n",
        "    rows, columns = confusion_matrix.shape\n",
        "    sum_of_recalls = 0\n",
        "    for label in range(columns):\n",
        "        sum_of_recalls += recall(label, confusion_matrix)\n",
        "    return sum_of_recalls / columns\n",
        "\n",
        "def f1score_macro_average(confusion_matrix):\n",
        "    rows, columns = confusion_matrix.shape\n",
        "    sum_of_f1score = 0\n",
        "    for label in range(columns):\n",
        "        print('label',label , ':', f1score(label, confusion_matrix))\n",
        "        sum_of_f1score += f1score(label, confusion_matrix)\n",
        "    return sum_of_f1score / columns\n",
        "\n",
        "def accuracy(confusion_matrix):\n",
        "    diagonal_sum = confusion_matrix.trace()\n",
        "    sum_of_all_elements = confusion_matrix.sum()\n",
        "    return diagonal_sum / sum_of_all_elements "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWIPQ-uJAscA"
      },
      "source": [
        "accuracy(cm)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqFnes8DAub-"
      },
      "source": [
        "print(\"precision total:\", precision_macro_average(cm))\n",
        "\n",
        "print(\"recall total:\", recall_macro_average(cm))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3InkWng-CNmy"
      },
      "source": [
        "print(\"f1score total:\", f1score_macro_average(cm))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}