digit_recognizer_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_DATA/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_DATA/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_DATA/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_DATA/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_DATA\", one_hot = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions to Create Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(shape):\n",
    "    init_random_dist = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(init_random_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_bias(shape):\n",
    "    init_random_bias = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(init_random_bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv2d(x,W):\n",
    "    # Creating a convolutional Neural Network\n",
    "    # x ----> [batch, H, W, Channel]\n",
    "    # W ----> [filter H, filter w, channels IN, channel out]\n",
    "    return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding=\"SAME\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_pool_2by2(x):\n",
    "    # function for max pooling\n",
    "    # x ----> [batch, H, W, Channel]\n",
    "    return tf.nn.max_pool(x, ksize=[1,2,2,1] , strides = [1,2,2,1], padding=\"SAME\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convolutional_layer(input_x, shape):\n",
    "    # returning the output after passing the Convolutional Neural Network to Relu Activation Function\n",
    "    W = init_weights(shape)\n",
    "    b = init_bias([shape[3]])\n",
    "    return tf.nn.relu(conv2d(input_x, W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal_full_layer(input_layer, size):\n",
    "    # Fully connected layer for last\n",
    "    input_size = int(input_layer.get_shape()[1])\n",
    "    W = init_weights([input_size, size])\n",
    "    b = init_bias([size])\n",
    "    return tf.matmul(input_layer, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Placeholders for our input x and output y\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "y_true = tf.placeholder(tf.float32, shape=[None,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshaping our input x (2-D) into our accepted input for CNN Network that is 4-D\n",
    "x_image = tf.reshape(x, shape=[-1,28,28,1]) # 784 = 28*28"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "convo1 = convolutional_layer(x_image, shape = [5,5,1,32])\n",
    "convo_1_pooling = max_pool_2by2(convo1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "convo_2 = convolutional_layer(convo_1_pooling,shape=[6,6,32,64])\n",
    "convo_2_pooling = max_pool_2by2(convo_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After first max pooling : 28/2 = 14\n",
    "# After second max pooling : 14/2 = 7\n",
    "convo_2_flat = tf.reshape(convo_2_pooling,[-1,7*7*64])\n",
    "full_layer_one = tf.nn.relu(normal_full_layer(convo_2_flat,1024))\n",
    "#1024 is the nos. of neurons we want in our fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hold_prob = tf.placeholder(tf.float32)\n",
    "full_one_dropout = tf.nn.dropout(full_layer_one,keep_prob=hold_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = normal_full_layer(full_one_dropout,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dimensions after different layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input Size:  (?, 784)\n",
      "After reshaping, input Size:  (?, 28, 28, 1)\n",
      "After first conolution:  (?, 28, 28, 32)\n",
      "After first Pooling:  (?, 14, 14, 32)\n",
      "After second conolution:  (?, 14, 14, 64)\n",
      "After second Pooling:  (?, 7, 7, 64)\n",
      "After flatening:  (?, 3136)\n",
      "After first fully dense NN:  (?, 1024)\n",
      "After first dropout:  (?, 1024)\n",
      "Prediction:  (?, 10)\n"
     ]
    }
   ],
   "source": [
    "print(\"input Size: \", x.get_shape())\n",
    "print(\"After reshaping, input Size: \", x_image.get_shape())\n",
    "print(\"After first conolution: \", convo1.get_shape())\n",
    "print(\"After first Pooling: \", convo_1_pooling.get_shape())\n",
    "print(\"After second conolution: \", convo_2.get_shape())\n",
    "print(\"After second Pooling: \", convo_2_pooling.get_shape())\n",
    "print(\"After flatening: \",convo_2_flat.get_shape())\n",
    "print(\"After first fully dense NN: \",full_layer_one.get_shape())\n",
    "print(\"After first dropout: \",full_one_dropout.get_shape())\n",
    "print(\"Prediction: \", y_pred.get_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_true,logits=y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)\n",
    "train = optimizer.minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [08:41<00:00,  9.50it/s]\n"
     ]
    }
   ],
   "source": [
    "epochs = 5000\n",
    "for i in tqdm(range(epochs)):\n",
    "\n",
    "    batch_x , batch_y = mnist.train.next_batch(50)\n",
    "\n",
    "    sess.run(train,feed_dict={x:batch_x,y_true:batch_y,hold_prob:0.5})\n",
    "\n",
    "    \n",
    "#     print('Currently on step {}'.format(i))\n",
    "#     print('Accuracy is:')\n",
    "    \n",
    "#     # Test the Train Model\n",
    "#     matches = tf.equal(tf.argmax(y_pred,1),tf.argmax(y_true,1))\n",
    "#     acc = tf.reduce_mean(tf.cast(matches,tf.float32))\n",
    "#     print(sess.run(acc,feed_dict={x:mnist.test.images,y_true:mnist.test.labels,hold_prob:1.0}))\n",
    "#     print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./CNN Model/CNN Model'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To restore the previous weights run the next command\n",
    "saver.save(sess, \"./CNN Model/CNN Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./CNN Model/CNN Model\n"
     ]
    }
   ],
   "source": [
    "saver.restore(sess, \"./CNN Model/CNN Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9939\n"
     ]
    }
   ],
   "source": [
    "matches = tf.equal(tf.argmax(y_pred,1),tf.argmax(y_true,1))\n",
    "acc = tf.reduce_mean(tf.cast(matches,tf.float32))\n",
    "print(sess.run(acc,feed_dict={x:mnist.test.images,y_true:mnist.test.labels,hold_prob:1.0}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 99.39%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "###### Accuracy is after almost 20000 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}