2_CNN_single.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'SVHN.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  #train_dataset = save['train_dataset']\n",
    "  train_labels = save['train_labels']\n",
    "  valid_dataset = save['valid_dataset']\n",
    "  valid_labels = save['valid_labels']\n",
    "  test_dataset = save['test_dataset']\n",
    "  test_labels = save['test_labels']\n",
    "  del save  # hint to help gc free up memory\n",
    "  #print('Training set', train_dataset.shape, train_labels.shape)\n",
    "  #print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "  #print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle_file = 'SVHN1.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset1 = save['train_dataset1']\n",
    "  del save  # hint to help gc free up memory\n",
    "\n",
    "pickle_file = 'SVHN2.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset2 = save['train_dataset2']\n",
    "  del save  # hint to help gc free up memory\n",
    "\n",
    "pickle_file = 'SVHN3.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset3 = save['train_dataset3']\n",
    "  del save  # hint to help gc free up memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (598388, 32, 32) (598388,)\n",
      "Validation set (6000, 32, 32) (6000,)\n",
      "Test set (26032, 32, 32) (26032,)\n"
     ]
    }
   ],
   "source": [
    "train_dataset = np.concatenate((train_dataset1, train_dataset2, train_dataset3), axis=0)\n",
    "del train_dataset1, train_dataset2, train_dataset3\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (598388, 32, 32, 1) (598388,)\n",
      "Validation set (6000, 32, 32, 1) (6000,)\n",
      "Test set (26032, 32, 32, 1) (26032,)\n"
     ]
    }
   ],
   "source": [
    "image_size = 32\n",
    "num_labels = 10\n",
    "num_channels = 1 # grayscale\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "  dataset = dataset.reshape(\n",
    "    (-1, image_size, image_size, num_channels)).astype(np.float32)\n",
    "  labels = labels.astype(np.int32)\n",
    "  return dataset, labels\n",
    "\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LecunLCN(X, image_shape, threshold=1e-4, radius=7, use_divisor=True):\n",
    "    \"\"\"Local Contrast Normalization\"\"\"\n",
    "    \"\"\"[http://yann.lecun.com/exdb/publis/pdf/jarrett-iccv-09.pdf]\"\"\"\n",
    "\n",
    "    # Get Gaussian filter\n",
    "    filter_shape = (radius, radius, image_shape[3], 1)\n",
    "\n",
    "    #self.filters = theano.shared(self.gaussian_filter(filter_shape), borrow=True)\n",
    "    filters = gaussian_filter(filter_shape)\n",
    "    X = tf.convert_to_tensor(X, dtype=tf.float32)\n",
    "    # Compute the Guassian weighted average by means of convolution\n",
    "    convout = tf.nn.conv2d(X, filters, [1,1,1,1], 'SAME')\n",
    "\n",
    "    # Subtractive step\n",
    "    mid = int(np.floor(filter_shape[1] / 2.))\n",
    "\n",
    "    # Make filter dimension broadcastable and subtract\n",
    "    centered_X = tf.sub(X, convout)\n",
    "\n",
    "    # Boolean marks whether or not to perform divisive step\n",
    "    if use_divisor:\n",
    "        # Note that the local variances can be computed by using the centered_X\n",
    "        # tensor. If we convolve this with the mean filter, that should give us\n",
    "        # the variance at each point. We simply take the square root to get our\n",
    "        # denominator\n",
    "\n",
    "        # Compute variances\n",
    "        sum_sqr_XX = tf.nn.conv2d(tf.square(centered_X), filters, [1,1,1,1], 'SAME')\n",
    "\n",
    "        # Take square root to get local standard deviation\n",
    "        denom = tf.sqrt(sum_sqr_XX)\n",
    "\n",
    "        per_img_mean = tf.reduce_mean(denom)\n",
    "        divisor = tf.maximum(per_img_mean, denom)\n",
    "        # Divisise step\n",
    "        new_X = tf.truediv(centered_X, tf.maximum(divisor, threshold))\n",
    "    else:\n",
    "        new_X = centered_X\n",
    "\n",
    "    return new_X\n",
    "\n",
    "\n",
    "def gaussian_filter(kernel_shape):\n",
    "    x = np.zeros(kernel_shape, dtype = float)\n",
    "    mid = np.floor(kernel_shape[0] / 2.)\n",
    "    \n",
    "    for kernel_idx in xrange(0, kernel_shape[2]):\n",
    "        for i in xrange(0, kernel_shape[0]):\n",
    "            for j in xrange(0, kernel_shape[1]):\n",
    "                x[i, j, kernel_idx, 0] = gauss(i - mid, j - mid)\n",
    "    \n",
    "    return tf.convert_to_tensor(x / np.sum(x), dtype=tf.float32)\n",
    "\n",
    "def gauss(x, y, sigma=3.0):\n",
    "    Z = 2 * np.pi * sigma ** 2\n",
    "    return  1. / Z * np.exp(-(x ** 2 + y ** 2) / (2. * sigma ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == labels)\n",
    "          / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.453111\n",
      "Minibatch accuracy: 12.5%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 500: 2.262278\n",
      "Minibatch accuracy: 12.5%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 1000: 2.193046\n",
      "Minibatch accuracy: 25.0%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 1500: 2.287955\n",
      "Minibatch accuracy: 15.6%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 2000: 1.888970\n",
      "Minibatch accuracy: 29.7%\n",
      "Validation accuracy: 19.6%\n",
      "Minibatch loss at step 2500: 0.379708\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 3000: 0.377859\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 3500: 0.319197\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 87.0%\n",
      "Minibatch loss at step 4000: 0.339692\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 87.5%\n",
      "Minibatch loss at step 4500: 0.216875\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 87.9%\n",
      "Minibatch loss at step 5000: 0.217608\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 88.3%\n",
      "Minibatch loss at step 5500: 0.263328\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 88.8%\n",
      "Minibatch loss at step 6000: 0.214657\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 89.1%\n",
      "Minibatch loss at step 6500: 0.347831\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 89.4%\n",
      "Minibatch loss at step 7000: 0.377402\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 89.6%\n",
      "Minibatch loss at step 7500: 0.092847\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 89.7%\n",
      "Minibatch loss at step 8000: 0.322605\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 90.0%\n",
      "Minibatch loss at step 8500: 0.255789\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 90.1%\n",
      "Minibatch loss at step 9000: 0.282733\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 90.3%\n",
      "Minibatch loss at step 9500: 0.320829\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 90.3%\n",
      "Minibatch loss at step 10000: 0.061666\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 90.8%\n",
      "Minibatch loss at step 10500: 0.100363\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 91.1%\n",
      "Minibatch loss at step 11000: 0.109308\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 90.8%\n",
      "Minibatch loss at step 11500: 0.181056\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 91.2%\n",
      "Minibatch loss at step 12000: 0.174621\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 91.2%\n",
      "Minibatch loss at step 12500: 0.223956\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 91.3%\n",
      "Minibatch loss at step 13000: 0.040920\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 91.3%\n",
      "Minibatch loss at step 13500: 0.103418\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 91.5%\n",
      "Minibatch loss at step 14000: 0.149366\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 91.4%\n",
      "Minibatch loss at step 14500: 0.132520\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 90.7%\n",
      "Minibatch loss at step 15000: 0.139116\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 91.1%\n",
      "Minibatch loss at step 15500: 0.056868\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 91.7%\n",
      "Minibatch loss at step 16000: 0.426161\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 91.2%\n",
      "Minibatch loss at step 16500: 0.223125\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 91.8%\n",
      "Minibatch loss at step 17000: 0.119249\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 91.8%\n",
      "Minibatch loss at step 17500: 0.304376\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 91.4%\n",
      "Minibatch loss at step 18000: 0.213656\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.0%\n",
      "Minibatch loss at step 18500: 0.410714\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 92.1%\n",
      "Minibatch loss at step 19000: 0.101434\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.3%\n",
      "Minibatch loss at step 19500: 0.211025\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.1%\n",
      "Minibatch loss at step 20000: 0.131821\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 20500: 0.148123\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.2%\n",
      "Minibatch loss at step 21000: 0.112688\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.2%\n",
      "Minibatch loss at step 21500: 0.127989\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.1%\n",
      "Minibatch loss at step 22000: 0.107971\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.0%\n",
      "Minibatch loss at step 22500: 0.391877\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.4%\n",
      "Minibatch loss at step 23000: 0.047703\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 23500: 0.208093\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 91.9%\n",
      "Minibatch loss at step 24000: 0.062427\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.4%\n",
      "Minibatch loss at step 24500: 0.251313\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 91.9%\n",
      "Minibatch loss at step 25000: 0.169109\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.3%\n",
      "Minibatch loss at step 25500: 0.098543\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.4%\n",
      "Minibatch loss at step 26000: 0.130105\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 26500: 0.043701\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 27000: 0.267333\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 27500: 0.228954\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 28000: 0.229270\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 28500: 0.111130\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 29000: 0.138779\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 29500: 0.068105\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.3%\n",
      "Minibatch loss at step 30000: 0.188792\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 30500: 0.177224\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 92.9%\n",
      "Minibatch loss at step 31000: 0.062245\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 31500: 0.062858\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.3%\n",
      "Minibatch loss at step 32000: 0.121681\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.3%\n",
      "Minibatch loss at step 32500: 0.117817\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.4%\n",
      "Minibatch loss at step 33000: 0.064135\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 33500: 0.169049\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 34000: 0.151915\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 34500: 0.138143\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 35000: 0.100215\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 35500: 0.026920\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 36000: 0.131434\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.9%\n",
      "Minibatch loss at step 36500: 0.263442\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 92.5%\n",
      "Minibatch loss at step 37000: 0.296210\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 37500: 0.078987\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 38000: 0.195182\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 38500: 0.200313\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 39000: 0.228058\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 39500: 0.091812\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 40000: 0.076461\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 40500: 0.291956\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 41000: 0.055680\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 41500: 0.183608\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 42000: 0.101181\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 42500: 0.128332\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 43000: 0.212580\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 43500: 0.027769\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 92.6%\n",
      "Minibatch loss at step 44000: 0.203329\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.9%\n",
      "Minibatch loss at step 44500: 0.083665\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.7%\n",
      "Minibatch loss at step 45000: 0.175870\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 45500: 0.027964\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 46000: 0.292265\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 46500: 0.257159\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 47000: 0.103929\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 47500: 0.056968\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 48000: 0.070906\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 48500: 0.029329\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 49000: 0.098648\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 49500: 0.073024\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 50000: 0.254658\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 50500: 0.160950\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 51000: 0.074162\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 51500: 0.093829\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 52000: 0.067329\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 52500: 0.012297\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 53000: 0.048955\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 53500: 0.173556\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 54000: 0.181573\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 54500: 0.045477\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 55000: 0.128254\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 55500: 0.143627\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 56000: 0.218192\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 56500: 0.019446\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 57000: 0.085973\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 57500: 0.157112\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 58000: 0.110420\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 58500: 0.094006\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 59000: 0.060276\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 59500: 0.030046\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 60000: 0.110089\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 60500: 0.282299\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 61000: 0.461526\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 61500: 0.050198\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 62000: 0.095896\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 62500: 0.099934\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 63000: 0.119063\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 63500: 0.096781\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 64000: 0.048758\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 64500: 0.435616\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 65000: 0.228071\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 65500: 0.046884\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 66000: 0.019537\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 66500: 0.237708\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 67000: 0.041182\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 67500: 0.019621\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 68000: 0.080172\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 68500: 0.134083\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.0%\n",
      "Minibatch loss at step 69000: 0.027825\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 69500: 0.103256\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 70000: 0.055942\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 70500: 0.036678\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 71000: 0.053889\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 71500: 0.063478\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 72000: 0.109585\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 72500: 0.079337\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 73000: 0.083104\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 73500: 0.130173\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 74000: 0.258995\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 93.1%\n",
      "Minibatch loss at step 74500: 0.098561\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 75000: 0.079453\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 75500: 0.014187\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 76000: 0.080112\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 76500: 0.076345\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 77000: 0.099522\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 77500: 0.017247\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 78000: 0.013133\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 78500: 0.195016\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 79000: 0.091172\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 79500: 0.101937\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 80000: 0.058824\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 80500: 0.109736\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 81000: 0.133990\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 81500: 0.134566\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 82000: 0.036682\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 82500: 0.046892\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 83000: 0.151699\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 83500: 0.217369\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 84000: 0.170973\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 84500: 0.063934\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 85000: 0.064542\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 85500: 0.076735\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 86000: 0.050374\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 86500: 0.143780\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 87000: 0.074523\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 87500: 0.073258\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 88000: 0.107974\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 88500: 0.107632\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 89000: 0.073481\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 89500: 0.125383\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.2%\n",
      "Minibatch loss at step 90000: 0.144785\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 90500: 0.018383\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 91000: 0.031889\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 91500: 0.046784\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 92000: 0.070724\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 92500: 0.183769\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 93000: 0.427370\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 93500: 0.015510\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 94000: 0.030862\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 94500: 0.053757\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 95000: 0.324041\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.8%\n",
      "Minibatch loss at step 95500: 0.023103\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 96000: 0.050115\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.6%\n",
      "Minibatch loss at step 96500: 0.154071\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 97000: 0.058371\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Minibatch loss at step 97500: 0.014275\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.7%\n",
      "Minibatch loss at step 98000: 0.124698\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 98500: 0.097070\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 99000: 0.043676\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 99500: 0.106184\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 100000: 0.106766\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 93.3%\n",
      "Test accuracy: 93.2%\n",
      "Model saved in file: CNN_1.ckpt\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "patch_size = 5\n",
    "depth1 = 16\n",
    "depth2 = 32\n",
    "depth3 = 64\n",
    "num_hidden1 = 64\n",
    "num_hidden2 = 16\n",
    "shape=[batch_size, image_size, image_size, num_channels]\n",
    "\n",
    "# Construct a 7-layer CNN.\n",
    "# C1: convolutional layer, batch_size x 28 x 28 x 16, convolution size: 5 x 5 x 1 x 16\n",
    "# S2: sub-sampling layer, batch_size x 14 x 14 x 16\n",
    "# C3: convolutional layer, batch_size x 10 x 10 x 32, convolution size: 5 x 5 x 16 x 32\n",
    "# S4: sub-sampling layer, batch_size x 5 x 5 x 32\n",
    "# C5: convolutional layer, batch_size x 1 x 1 x 64, convolution size: 5 x 5 x 32 x 64\n",
    "# Dropout\n",
    "# F6: fully-connected layer, weight size: 64 x 16\n",
    "# Output layer, weight size: 16 x 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.int64, shape=(batch_size))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.get_variable(\"W1\", shape=[patch_size, patch_size, num_channels, depth1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  layer1_biases = tf.Variable(tf.constant(1.0, shape=[depth1]), name='B1')\n",
    "  layer2_weights = tf.get_variable(\"W2\", shape=[patch_size, patch_size, depth1, depth2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth2]), name='B2')\n",
    "  layer3_weights = tf.get_variable(\"W3\", shape=[patch_size, patch_size, depth2, num_hidden1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden1]), name='B3')\n",
    "  layer4_weights = tf.get_variable(\"W4\", shape=[num_hidden1, num_hidden2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='B4')\n",
    "  layer5_weights = tf.get_variable(\"W5\", shape=[num_hidden2, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer5_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='B5')\n",
    "  \n",
    "  # Model.\n",
    "  def model(data, keep_prob, shape):\n",
    "    LCN = LecunLCN(data, shape)\n",
    "    conv = tf.nn.conv2d(LCN, layer1_weights, [1,1,1,1], 'VALID', name='C1')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='S2')\n",
    "    conv = tf.nn.conv2d(sub, layer2_weights, [1,1,1,1], padding='VALID', name='C3')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='S4')\n",
    "    conv = tf.nn.conv2d(sub, layer3_weights, [1,1,1,1], padding='VALID', name='C5')\n",
    "    hidden = tf.nn.relu(conv + layer3_biases)\n",
    "    hidden = tf.nn.dropout(hidden, keep_prob)\n",
    "    shape = hidden.get_shape().as_list()\n",
    "    reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, layer4_weights) + layer4_biases)\n",
    "    return tf.matmul(hidden, layer5_weights) + layer5_biases\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset, 0.9375, shape)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "    \n",
    "  # Optimizer.\n",
    "  #optimizer = tf.train.AdagradOptimizer(0.01).minimize(loss)\n",
    "  global_step = tf.Variable(0)\n",
    "  learning_rate = tf.train.exponential_decay(0.05, global_step, 10000, 0.95)\n",
    "  optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(model(tf_train_dataset, 1.0, shape))\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset, 1.0, shape))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset, 1.0, shape))\n",
    "    \n",
    "  saver = tf.train.Saver()\n",
    "\n",
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size)]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    \n",
    "    if (step % 500 == 0): \n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))\n",
    "  save_path = saver.save(session, \"CNN_1.ckpt\")\n",
    "  print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model restored.\n",
      "Initialized\n",
      "Minibatch loss at step 0: 3.212271\n",
      "Minibatch accuracy: 9.4%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 500: 2.268531\n",
      "Minibatch accuracy: 12.5%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 1000: 2.187550\n",
      "Minibatch accuracy: 25.0%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 1500: 2.286384\n",
      "Minibatch accuracy: 15.6%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 2000: 2.227596\n",
      "Minibatch accuracy: 15.6%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 2500: 2.332437\n",
      "Minibatch accuracy: 9.4%\n",
      "Validation accuracy: 10.0%\n",
      "Minibatch loss at step 3000: 2.257286\n",
      "Minibatch accuracy: 17.2%\n",
      "Validation accuracy: 10.0%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-49-13ce8a7e0a17>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    162\u001b[0m     \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mtf_train_dataset\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mbatch_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf_train_labels\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m     _, l, predictions = session.run(\n\u001b[0;32m--> 164\u001b[0;31m       [optimizer, loss, train_prediction], feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m    165\u001b[0m     \u001b[0;31m#print(predictions[:])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m500\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/HangYao/anaconda/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    370\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 372\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    373\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/HangYao/anaconda/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    634\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m       results = self._do_run(handle, target_list, unique_fetches,\n\u001b[0;32m--> 636\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    637\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m       \u001b[0;31m# The movers are no longer used. Delete them.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/HangYao/anaconda/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    706\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    707\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m--> 708\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m    709\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    710\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/HangYao/anaconda/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    713\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    714\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 715\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    716\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    717\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/HangYao/anaconda/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m    695\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m    696\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 697\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m    698\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    699\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "patch_size1 = 5\n",
    "patch_size2 = 3\n",
    "patch_size3 = 1\n",
    "depth1 = 16\n",
    "depth2 = 32\n",
    "depth3 = 96\n",
    "num_hidden1 = 64\n",
    "num_hidden2 = 32\n",
    "shape=[batch_size, image_size, image_size, num_channels]\n",
    "\n",
    "# Construct a Inception Module CNN.\n",
    "# C1: convolutional layer, batch_size x 28 x 28 x 16, convolution size: 5 x 5 x 1 x 16\n",
    "# P2: max pool layer, batch_size x 14 x 14 x 16\n",
    "# C3: convolutional layer, batch_size x 10 x 10 x 32, convolution size: 5 x 5 x 16 x 32\n",
    "\n",
    "# Inception Module:\n",
    "# Inception & Concat: output batch_size x 5 x 5 x 96\n",
    "# C6: convolutional layer, batch_size x 5 x 5 x 32, convolution size: 1 x 1 x 96 x 32\n",
    "# C7: convolutional layer, batch_size x 1 x 1 x 64, convolution size: 5 x 5 x 32 x 64\n",
    "# Dropout layer\n",
    "# F8: fully-connected layer, weight size: 64 x 16\n",
    "# Output layer, weight size: 16 x 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  tf_train_dataset = tf.placeholder(\n",
    "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "  tf_train_labels = tf.placeholder(tf.int64, shape=(batch_size))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  layer1_weights = tf.get_variable(\"W1\", shape=[patch_size1, patch_size1, num_channels, depth1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  layer1_biases = tf.Variable(tf.constant(1.0, shape=[depth1]), name='B1')\n",
    "\n",
    "  layer2_weights = tf.get_variable(\"W2\", shape=[patch_size1, patch_size1, depth1, depth2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth2]), name='B2')\n",
    "\n",
    "  layer6_weights = tf.get_variable(\"W6\", shape=[patch_size3, patch_size3, depth3, depth2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer6_biases = tf.Variable(tf.constant(1.0, shape=[depth2]), name='B6')\n",
    "\n",
    "  layer3_weights = tf.get_variable(\"W3\", shape=[patch_size1, patch_size1, depth2, num_hidden1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden1]), name='B3')\n",
    "\n",
    "  layer4_weights = tf.get_variable(\"W4\", shape=[num_hidden1, num_hidden2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='B4')\n",
    "\n",
    "  layer5_weights = tf.get_variable(\"W5\", shape=[num_hidden2, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "  layer5_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='B5')\n",
    "    \n",
    "  # Inception Variables.\n",
    "  incep_11_w = tf.get_variable(\"I11W\", shape=[1, 1, depth2, 16],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_11_b = tf.Variable(tf.constant(1.0, shape=[16]), name='I11B')\n",
    "\n",
    "  incep_12_w = tf.get_variable(\"I12W\", shape=[patch_size1, patch_size1, 16, 32],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_12_b = tf.Variable(tf.constant(1.0, shape=[32]), name='I12B')\n",
    "\n",
    "  incep_21_w = tf.get_variable(\"I21W\", shape=[1, 1, depth2, 16],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_21_b = tf.Variable(tf.constant(1.0, shape=[16]), name='I21B')\n",
    "\n",
    "  incep_22_w = tf.get_variable(\"I22W\", shape=[patch_size2, patch_size2, 16, 32],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_22_b = tf.Variable(tf.constant(1.0, shape=[32]), name='I22B')\n",
    "\n",
    "  incep_31_w = tf.get_variable(\"I31W\", shape=[1, 1, depth2, 16],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_31_b = tf.Variable(tf.constant(1.0, shape=[16]), name='I31B')\n",
    "\n",
    "  incep_41_w = tf.get_variable(\"I41W\", shape=[1, 1, depth2, 16],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "  incep_41_b = tf.Variable(tf.constant(1.0, shape=[16]), name='I41B')\n",
    "    \n",
    "  # Model.\n",
    "  def model(data, keep_prob, shape):\n",
    "    LCN = LecunLCN(data, shape)\n",
    "    conv = tf.nn.conv2d(LCN, layer1_weights, [1,1,1,1], 'VALID', name='C1')\n",
    "    hidden = tf.nn.relu(conv + layer1_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='P2')\n",
    "    conv = tf.nn.conv2d(sub, layer2_weights, [1,1,1,1], padding='VALID', name='C3')\n",
    "    hidden = tf.nn.relu(conv + layer2_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    \n",
    "    # Inception Module\n",
    "    hidden = Inception(lrn)\n",
    "\n",
    "    conv = tf.nn.conv2d(hidden, layer6_weights, [1,1,1,1], padding='SAME', name='C6')\n",
    "    hidden = tf.nn.relu(conv + layer6_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    conv = tf.nn.conv2d(lrn, layer3_weights, [1,1,1,1], padding='VALID', name='C7')\n",
    "    hidden = tf.nn.relu(conv + layer3_biases)\n",
    "    lrn = tf.nn.local_response_normalization(hidden)\n",
    "    hidden = tf.nn.dropout(lrn, keep_prob)\n",
    "    shape = hidden.get_shape().as_list()\n",
    "    reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])    \n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, layer4_weights) + layer4_biases)\n",
    "    return tf.matmul(hidden, layer5_weights) + layer5_biases\n",
    "\n",
    "  def Inception(input_tensor):\n",
    "        # Branch 1\n",
    "        conv1 = tf.nn.conv2d(input_tensor, incep_11_w, [1,1,1,1], padding='SAME', name='I1C1')\n",
    "        hidden1 = tf.nn.relu(conv1 + incep_11_b)\n",
    "        lrn1 = tf.nn.local_response_normalization(hidden1)\n",
    "        conv1 = tf.nn.conv2d(lrn1, incep_12_w, [1,2,2,1], padding='SAME', name='I1C2')\n",
    "        hidden1 = tf.nn.relu(conv1 + incep_12_b)\n",
    "        lrn1 = tf.nn.local_response_normalization(hidden1)\n",
    "        # Branch 2\n",
    "        conv2 = tf.nn.conv2d(input_tensor, incep_21_w, [1,1,1,1], padding='SAME', name='I2C1')\n",
    "        hidden2 = tf.nn.relu(conv2 + incep_21_b)\n",
    "        lrn2 = tf.nn.local_response_normalization(hidden2)\n",
    "        conv2 = tf.nn.conv2d(lrn2, incep_22_w, [1,2,2,1], padding='SAME', name='I2C2')\n",
    "        hidden2 = tf.nn.relu(conv2 + incep_22_b)\n",
    "        lrn2 = tf.nn.local_response_normalization(hidden2)\n",
    "        # Branch 3\n",
    "        sub3 = tf.nn.avg_pool(input_tensor, [1,3,3,1], [1,2,2,1], 'SAME', name='I3P1')\n",
    "        conv3 = tf.nn.conv2d(sub3, incep_31_w, [1,1,1,1], padding='SAME', name='I3C2')\n",
    "        hidden3 = tf.nn.relu(conv3 + incep_31_b)\n",
    "        lrn3 = tf.nn.local_response_normalization(hidden3)\n",
    "        # Branch 4\n",
    "        conv4 = tf.nn.conv2d(input_tensor, incep_41_w, [1,2,2,1], padding='SAME', name='I4C1')\n",
    "        hidden4 = tf.nn.relu(conv4 + incep_41_b)\n",
    "        lrn4 = tf.nn.local_response_normalization(hidden4)\n",
    "        # Concat\n",
    "        output_tensor = tf.concat(3, [lrn1, lrn2, lrn3, lrn4], name='CONCAT')\n",
    "        return output_tensor\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = model(tf_train_dataset, 1, shape)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "    \n",
    "  # Optimizer.\n",
    "  #optimizer = tf.train.AdagradOptimizer(0.01).minimize(loss)\n",
    "  global_step = tf.Variable(0)\n",
    "  learning_rate = tf.train.exponential_decay(0.01, global_step, 10000, 0.95)\n",
    "  optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(model(tf_train_dataset, 1.0, shape))\n",
    "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset, 1.0, shape))\n",
    "  test_prediction = tf.nn.softmax(model(tf_test_dataset, 1.0, shape))\n",
    "    \n",
    "  saver = tf.train.Saver()\n",
    "\n",
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()  \n",
    "  reader = tf.train.NewCheckpointReader(\"CNN_1.ckpt\")\n",
    "  reader.get_variable_to_shape_map()\n",
    "  #saver.restore(session, \"CNN_1.ckpt\")\n",
    "  print(\"Model restored.\")  \n",
    "  \n",
    "  \n",
    "  #tf.initialize_all_variables().run()\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size)]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    #print(predictions[:])\n",
    "    if (step % 500 == 0): \n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))\n",
    "  save_path = saver.save(session, \"CNN_2.ckpt\")\n",
    "  print(\"Model saved in file: %s\" % save_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}