Nueral Network with Keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nueral Network with Keras Using MNIST data with one hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import print_function\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "np.random.seed(1671) # for reproducibility\n",
    "# import cv2\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata=pd.read_csv('mnist_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdata=pd.read_csv('mnist_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59999, 785)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traindata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9999, 785)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain=traindata.iloc[:,1:]\n",
    "ytrain=traindata.iloc[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>5</th>\n",
       "      <th>0</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.6</th>\n",
       "      <th>0.7</th>\n",
       "      <th>0.8</th>\n",
       "      <th>...</th>\n",
       "      <th>0.608</th>\n",
       "      <th>0.609</th>\n",
       "      <th>0.610</th>\n",
       "      <th>0.611</th>\n",
       "      <th>0.612</th>\n",
       "      <th>0.613</th>\n",
       "      <th>0.614</th>\n",
       "      <th>0.615</th>\n",
       "      <th>0.616</th>\n",
       "      <th>0.617</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   5  0  0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  ...    0.608  0.609  0.610  \\\n",
       "0  0  0    0    0    0    0    0    0    0    0  ...        0      0      0   \n",
       "1  4  0    0    0    0    0    0    0    0    0  ...        0      0      0   \n",
       "2  1  0    0    0    0    0    0    0    0    0  ...        0      0      0   \n",
       "3  9  0    0    0    0    0    0    0    0    0  ...        0      0      0   \n",
       "4  2  0    0    0    0    0    0    0    0    0  ...        0      0      0   \n",
       "\n",
       "   0.611  0.612  0.613  0.614  0.615  0.616  0.617  \n",
       "0      0      0      0      0      0      0      0  \n",
       "1      0      0      0      0      0      0      0  \n",
       "2      0      0      0      0      0      0      0  \n",
       "3      0      0      0      0      0      0      0  \n",
       "4      0      0      0      0      0      0      0  \n",
       "\n",
       "[5 rows x 785 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtest=testdata.iloc[:,1:]\n",
    "ytest=testdata.iloc[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59999"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_EPOCH = 200\n",
    "BATCH_SIZE = 64\n",
    "VERBOSE = 1\n",
    "NB_CLASSES = 10 # number of outputs = number of digits\n",
    "OPTIMIZER = SGD() # SGD optimizer, explained later in this chapter\n",
    "N_HIDDEN = 128\n",
    "VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION\n",
    "# data: shuffled and split between train and test sets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59999 train samples\n",
      "9999 test samples\n"
     ]
    }
   ],
   "source": [
    "RESHAPED = 784\n",
    "xtrain /= 255\n",
    "xtest /= 255\n",
    "print(xtrain.shape[0], 'train samples')\n",
    "print(xtest.shape[0], 'test samples')\n",
    "# convert class vectors to binary class matrices\n",
    "ytrain = np_utils.to_categorical(ytrain, NB_CLASSES)\n",
    "ytest = np_utils.to_categorical(ytest, NB_CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 10)                7850      \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 7,850\n",
      "Trainable params: 7,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 10 outputs\n",
    "# final stage is softmax\n",
    "model = Sequential()\n",
    "model.add(Dense(NB_CLASSES, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 47999 samples, validate on 12000 samples\n",
      "Epoch 1/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 1.0781 - accuracy: 0.7512 - val_loss: 0.6567 - val_accuracy: 0.8553\n",
      "Epoch 2/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.6081 - accuracy: 0.8552 - val_loss: 0.5103 - val_accuracy: 0.8760\n",
      "Epoch 3/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.5129 - accuracy: 0.8705 - val_loss: 0.4515 - val_accuracy: 0.8857\n",
      "Epoch 4/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.4665 - accuracy: 0.8782 - val_loss: 0.4192 - val_accuracy: 0.8916\n",
      "Epoch 5/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.4379 - accuracy: 0.8839 - val_loss: 0.3978 - val_accuracy: 0.8965\n",
      "Epoch 6/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.4179 - accuracy: 0.8878 - val_loss: 0.3831 - val_accuracy: 0.8983\n",
      "Epoch 7/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.4031 - accuracy: 0.8908 - val_loss: 0.3715 - val_accuracy: 0.9007\n",
      "Epoch 8/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.3914 - accuracy: 0.8936 - val_loss: 0.3622 - val_accuracy: 0.9023\n",
      "Epoch 9/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3819 - accuracy: 0.8952 - val_loss: 0.3546 - val_accuracy: 0.9042\n",
      "Epoch 10/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3740 - accuracy: 0.8975 - val_loss: 0.3483 - val_accuracy: 0.9051\n",
      "Epoch 11/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3673 - accuracy: 0.8987 - val_loss: 0.3433 - val_accuracy: 0.9061\n",
      "Epoch 12/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3615 - accuracy: 0.8999 - val_loss: 0.3387 - val_accuracy: 0.9067\n",
      "Epoch 13/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3564 - accuracy: 0.9014 - val_loss: 0.3346 - val_accuracy: 0.9081\n",
      "Epoch 14/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3518 - accuracy: 0.9022 - val_loss: 0.3313 - val_accuracy: 0.9093\n",
      "Epoch 15/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3478 - accuracy: 0.9034 - val_loss: 0.3277 - val_accuracy: 0.9094\n",
      "Epoch 16/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3441 - accuracy: 0.9043 - val_loss: 0.3253 - val_accuracy: 0.9097\n",
      "Epoch 17/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3409 - accuracy: 0.9052 - val_loss: 0.3223 - val_accuracy: 0.9109\n",
      "Epoch 18/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3377 - accuracy: 0.9063 - val_loss: 0.3201 - val_accuracy: 0.9111\n",
      "Epoch 19/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3350 - accuracy: 0.9070 - val_loss: 0.3178 - val_accuracy: 0.9120\n",
      "Epoch 20/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3324 - accuracy: 0.9074 - val_loss: 0.3160 - val_accuracy: 0.9124\n",
      "Epoch 21/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3301 - accuracy: 0.9082 - val_loss: 0.3139 - val_accuracy: 0.9133\n",
      "Epoch 22/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3278 - accuracy: 0.9091 - val_loss: 0.3124 - val_accuracy: 0.9131\n",
      "Epoch 23/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3258 - accuracy: 0.9093 - val_loss: 0.3105 - val_accuracy: 0.9140\n",
      "Epoch 24/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3238 - accuracy: 0.9098 - val_loss: 0.3092 - val_accuracy: 0.9148\n",
      "Epoch 25/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3220 - accuracy: 0.9105 - val_loss: 0.3077 - val_accuracy: 0.9146\n",
      "Epoch 26/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3203 - accuracy: 0.9110 - val_loss: 0.3062 - val_accuracy: 0.9143\n",
      "Epoch 27/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3186 - accuracy: 0.9114 - val_loss: 0.3051 - val_accuracy: 0.9153\n",
      "Epoch 28/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3172 - accuracy: 0.9117 - val_loss: 0.3039 - val_accuracy: 0.9158\n",
      "Epoch 29/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3157 - accuracy: 0.9124 - val_loss: 0.3028 - val_accuracy: 0.9154\n",
      "Epoch 30/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3143 - accuracy: 0.9126 - val_loss: 0.3018 - val_accuracy: 0.9152\n",
      "Epoch 31/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3129 - accuracy: 0.9129 - val_loss: 0.3008 - val_accuracy: 0.9156\n",
      "Epoch 32/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3117 - accuracy: 0.9136 - val_loss: 0.2998 - val_accuracy: 0.9160\n",
      "Epoch 33/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3105 - accuracy: 0.9134 - val_loss: 0.2989 - val_accuracy: 0.9168\n",
      "Epoch 34/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3094 - accuracy: 0.9138 - val_loss: 0.2979 - val_accuracy: 0.9165\n",
      "Epoch 35/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3082 - accuracy: 0.9140 - val_loss: 0.2974 - val_accuracy: 0.9163\n",
      "Epoch 36/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3072 - accuracy: 0.9142 - val_loss: 0.2965 - val_accuracy: 0.9168\n",
      "Epoch 37/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3062 - accuracy: 0.9149 - val_loss: 0.2955 - val_accuracy: 0.9172\n",
      "Epoch 38/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.3052 - accuracy: 0.9152 - val_loss: 0.2949 - val_accuracy: 0.9175\n",
      "Epoch 39/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3043 - accuracy: 0.9150 - val_loss: 0.2942 - val_accuracy: 0.9173\n",
      "Epoch 40/200\n",
      "47999/47999 [==============================] - 1s 27us/step - loss: 0.3033 - accuracy: 0.9157 - val_loss: 0.2937 - val_accuracy: 0.9178\n",
      "Epoch 41/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.3025 - accuracy: 0.9159 - val_loss: 0.2931 - val_accuracy: 0.9183\n",
      "Epoch 42/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.3016 - accuracy: 0.9157 - val_loss: 0.2925 - val_accuracy: 0.9184\n",
      "Epoch 43/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.3009 - accuracy: 0.9162 - val_loss: 0.2917 - val_accuracy: 0.9186\n",
      "Epoch 44/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.3001 - accuracy: 0.9164 - val_loss: 0.2913 - val_accuracy: 0.9191\n",
      "Epoch 45/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2993 - accuracy: 0.9165 - val_loss: 0.2910 - val_accuracy: 0.9192\n",
      "Epoch 46/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2986 - accuracy: 0.9166 - val_loss: 0.2900 - val_accuracy: 0.9197\n",
      "Epoch 47/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2979 - accuracy: 0.9170 - val_loss: 0.2897 - val_accuracy: 0.9191\n",
      "Epoch 48/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2972 - accuracy: 0.9174 - val_loss: 0.2890 - val_accuracy: 0.9197\n",
      "Epoch 49/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2965 - accuracy: 0.9174 - val_loss: 0.2889 - val_accuracy: 0.9199\n",
      "Epoch 50/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2958 - accuracy: 0.9177 - val_loss: 0.2882 - val_accuracy: 0.9195\n",
      "Epoch 51/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2952 - accuracy: 0.9176 - val_loss: 0.2878 - val_accuracy: 0.9199\n",
      "Epoch 52/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2946 - accuracy: 0.9180 - val_loss: 0.2874 - val_accuracy: 0.9205\n",
      "Epoch 53/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2940 - accuracy: 0.9182 - val_loss: 0.2869 - val_accuracy: 0.9204\n",
      "Epoch 54/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2934 - accuracy: 0.9185 - val_loss: 0.2866 - val_accuracy: 0.9205\n",
      "Epoch 55/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2928 - accuracy: 0.9181 - val_loss: 0.2861 - val_accuracy: 0.9207\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2923 - accuracy: 0.9182 - val_loss: 0.2857 - val_accuracy: 0.9210\n",
      "Epoch 57/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2918 - accuracy: 0.9189 - val_loss: 0.2855 - val_accuracy: 0.9211\n",
      "Epoch 58/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2912 - accuracy: 0.9188 - val_loss: 0.2848 - val_accuracy: 0.9212\n",
      "Epoch 59/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2907 - accuracy: 0.9187 - val_loss: 0.2846 - val_accuracy: 0.9212\n",
      "Epoch 60/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2902 - accuracy: 0.9192 - val_loss: 0.2845 - val_accuracy: 0.9208\n",
      "Epoch 61/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2897 - accuracy: 0.9190 - val_loss: 0.2840 - val_accuracy: 0.9212\n",
      "Epoch 62/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2893 - accuracy: 0.9194 - val_loss: 0.2837 - val_accuracy: 0.9214\n",
      "Epoch 63/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2888 - accuracy: 0.9194 - val_loss: 0.2834 - val_accuracy: 0.9214\n",
      "Epoch 64/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2883 - accuracy: 0.9196 - val_loss: 0.2832 - val_accuracy: 0.9214\n",
      "Epoch 65/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2879 - accuracy: 0.9197 - val_loss: 0.2828 - val_accuracy: 0.9213\n",
      "Epoch 66/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2875 - accuracy: 0.9197 - val_loss: 0.2825 - val_accuracy: 0.9212\n",
      "Epoch 67/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2871 - accuracy: 0.9200 - val_loss: 0.2821 - val_accuracy: 0.9215\n",
      "Epoch 68/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2866 - accuracy: 0.9200 - val_loss: 0.2818 - val_accuracy: 0.9218\n",
      "Epoch 69/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2862 - accuracy: 0.9199 - val_loss: 0.2816 - val_accuracy: 0.9217\n",
      "Epoch 70/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2858 - accuracy: 0.9196 - val_loss: 0.2814 - val_accuracy: 0.9215\n",
      "Epoch 71/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2854 - accuracy: 0.9199 - val_loss: 0.2812 - val_accuracy: 0.9219\n",
      "Epoch 72/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2850 - accuracy: 0.9202 - val_loss: 0.2809 - val_accuracy: 0.9215\n",
      "Epoch 73/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2847 - accuracy: 0.9206 - val_loss: 0.2806 - val_accuracy: 0.9222\n",
      "Epoch 74/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2842 - accuracy: 0.9207 - val_loss: 0.2804 - val_accuracy: 0.9222\n",
      "Epoch 75/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2839 - accuracy: 0.9209 - val_loss: 0.2801 - val_accuracy: 0.9222\n",
      "Epoch 76/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2836 - accuracy: 0.9206 - val_loss: 0.2799 - val_accuracy: 0.9227\n",
      "Epoch 77/200\n",
      "47999/47999 [==============================] - 1s 28us/step - loss: 0.2832 - accuracy: 0.9211 - val_loss: 0.2796 - val_accuracy: 0.9222\n",
      "Epoch 78/200\n",
      "47999/47999 [==============================] - 1s 29us/step - loss: 0.2829 - accuracy: 0.9211 - val_loss: 0.2795 - val_accuracy: 0.9223\n",
      "Epoch 79/200\n",
      "47999/47999 [==============================] - 1s 28us/step - loss: 0.2825 - accuracy: 0.9210 - val_loss: 0.2792 - val_accuracy: 0.9227\n",
      "Epoch 80/200\n",
      "47999/47999 [==============================] - 1s 27us/step - loss: 0.2821 - accuracy: 0.9211 - val_loss: 0.2791 - val_accuracy: 0.9224\n",
      "Epoch 81/200\n",
      "47999/47999 [==============================] - 1s 28us/step - loss: 0.2819 - accuracy: 0.9211 - val_loss: 0.2788 - val_accuracy: 0.9223\n",
      "Epoch 82/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2815 - accuracy: 0.9215 - val_loss: 0.2787 - val_accuracy: 0.9222\n",
      "Epoch 83/200\n",
      "47999/47999 [==============================] - 1s 27us/step - loss: 0.2812 - accuracy: 0.9213 - val_loss: 0.2785 - val_accuracy: 0.9229\n",
      "Epoch 84/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2809 - accuracy: 0.9217 - val_loss: 0.2784 - val_accuracy: 0.9228\n",
      "Epoch 85/200\n",
      "47999/47999 [==============================] - 1s 27us/step - loss: 0.2807 - accuracy: 0.9216 - val_loss: 0.2781 - val_accuracy: 0.9227\n",
      "Epoch 86/200\n",
      "47999/47999 [==============================] - 1s 28us/step - loss: 0.2802 - accuracy: 0.9217 - val_loss: 0.2781 - val_accuracy: 0.9222\n",
      "Epoch 87/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2800 - accuracy: 0.9216 - val_loss: 0.2778 - val_accuracy: 0.9230\n",
      "Epoch 88/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2797 - accuracy: 0.9220 - val_loss: 0.2775 - val_accuracy: 0.9230\n",
      "Epoch 89/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2794 - accuracy: 0.9220 - val_loss: 0.2776 - val_accuracy: 0.9227\n",
      "Epoch 90/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2791 - accuracy: 0.9221 - val_loss: 0.2772 - val_accuracy: 0.9238\n",
      "Epoch 91/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2788 - accuracy: 0.9222 - val_loss: 0.2772 - val_accuracy: 0.9233\n",
      "Epoch 92/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2785 - accuracy: 0.9223 - val_loss: 0.2768 - val_accuracy: 0.9234\n",
      "Epoch 93/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2783 - accuracy: 0.9223 - val_loss: 0.2768 - val_accuracy: 0.9234\n",
      "Epoch 94/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2781 - accuracy: 0.9222 - val_loss: 0.2766 - val_accuracy: 0.9233\n",
      "Epoch 95/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2778 - accuracy: 0.9226 - val_loss: 0.2767 - val_accuracy: 0.9235\n",
      "Epoch 96/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2775 - accuracy: 0.9224 - val_loss: 0.2763 - val_accuracy: 0.9231\n",
      "Epoch 97/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2773 - accuracy: 0.9226 - val_loss: 0.2761 - val_accuracy: 0.9236\n",
      "Epoch 98/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2770 - accuracy: 0.9226 - val_loss: 0.2760 - val_accuracy: 0.9243\n",
      "Epoch 99/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2768 - accuracy: 0.9228 - val_loss: 0.2759 - val_accuracy: 0.9233\n",
      "Epoch 100/200\n",
      "47999/47999 [==============================] - 1s 26us/step - loss: 0.2765 - accuracy: 0.9230 - val_loss: 0.2758 - val_accuracy: 0.9235\n",
      "Epoch 101/200\n",
      "47999/47999 [==============================] - 1s 29us/step - loss: 0.2763 - accuracy: 0.9230 - val_loss: 0.2756 - val_accuracy: 0.9239\n",
      "Epoch 102/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2760 - accuracy: 0.9234 - val_loss: 0.2754 - val_accuracy: 0.9241\n",
      "Epoch 103/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2758 - accuracy: 0.9230 - val_loss: 0.2751 - val_accuracy: 0.9242\n",
      "Epoch 104/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2756 - accuracy: 0.9232 - val_loss: 0.2751 - val_accuracy: 0.9237\n",
      "Epoch 105/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2754 - accuracy: 0.9234 - val_loss: 0.2750 - val_accuracy: 0.9245\n",
      "Epoch 106/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2751 - accuracy: 0.9235 - val_loss: 0.2750 - val_accuracy: 0.9237\n",
      "Epoch 107/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2749 - accuracy: 0.9235 - val_loss: 0.2749 - val_accuracy: 0.9239\n",
      "Epoch 108/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2747 - accuracy: 0.9232 - val_loss: 0.2749 - val_accuracy: 0.9236\n",
      "Epoch 109/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2745 - accuracy: 0.9234 - val_loss: 0.2745 - val_accuracy: 0.9243\n",
      "Epoch 110/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2742 - accuracy: 0.9236 - val_loss: 0.2744 - val_accuracy: 0.9243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 111/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2740 - accuracy: 0.9236 - val_loss: 0.2743 - val_accuracy: 0.9244\n",
      "Epoch 112/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2738 - accuracy: 0.9238 - val_loss: 0.2741 - val_accuracy: 0.9247\n",
      "Epoch 113/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2736 - accuracy: 0.9236 - val_loss: 0.2740 - val_accuracy: 0.9247\n",
      "Epoch 114/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2734 - accuracy: 0.9240 - val_loss: 0.2741 - val_accuracy: 0.9245\n",
      "Epoch 115/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2732 - accuracy: 0.9237 - val_loss: 0.2740 - val_accuracy: 0.9239\n",
      "Epoch 116/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2731 - accuracy: 0.9237 - val_loss: 0.2738 - val_accuracy: 0.9243\n",
      "Epoch 117/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2728 - accuracy: 0.9238 - val_loss: 0.2739 - val_accuracy: 0.9243\n",
      "Epoch 118/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2726 - accuracy: 0.9240 - val_loss: 0.2736 - val_accuracy: 0.9246\n",
      "Epoch 119/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2723 - accuracy: 0.9244 - val_loss: 0.2735 - val_accuracy: 0.9248\n",
      "Epoch 120/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2722 - accuracy: 0.9243 - val_loss: 0.2733 - val_accuracy: 0.9245\n",
      "Epoch 121/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2720 - accuracy: 0.9241 - val_loss: 0.2734 - val_accuracy: 0.9247\n",
      "Epoch 122/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2718 - accuracy: 0.9244 - val_loss: 0.2732 - val_accuracy: 0.9249\n",
      "Epoch 123/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2717 - accuracy: 0.9245 - val_loss: 0.2732 - val_accuracy: 0.9243\n",
      "Epoch 124/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2715 - accuracy: 0.9245 - val_loss: 0.2729 - val_accuracy: 0.9247\n",
      "Epoch 125/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2713 - accuracy: 0.9246 - val_loss: 0.2728 - val_accuracy: 0.9247\n",
      "Epoch 126/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2711 - accuracy: 0.9247 - val_loss: 0.2728 - val_accuracy: 0.9245\n",
      "Epoch 127/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2709 - accuracy: 0.9243 - val_loss: 0.2728 - val_accuracy: 0.9243\n",
      "Epoch 128/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2708 - accuracy: 0.9247 - val_loss: 0.2727 - val_accuracy: 0.9247\n",
      "Epoch 129/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2705 - accuracy: 0.9247 - val_loss: 0.2726 - val_accuracy: 0.9245\n",
      "Epoch 130/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2704 - accuracy: 0.9250 - val_loss: 0.2724 - val_accuracy: 0.9248\n",
      "Epoch 131/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2703 - accuracy: 0.9247 - val_loss: 0.2723 - val_accuracy: 0.9245\n",
      "Epoch 132/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2701 - accuracy: 0.9250 - val_loss: 0.2723 - val_accuracy: 0.9247\n",
      "Epoch 133/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2699 - accuracy: 0.9252 - val_loss: 0.2722 - val_accuracy: 0.9243\n",
      "Epoch 134/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2697 - accuracy: 0.9250 - val_loss: 0.2721 - val_accuracy: 0.9248\n",
      "Epoch 135/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2696 - accuracy: 0.9249 - val_loss: 0.2720 - val_accuracy: 0.9246\n",
      "Epoch 136/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2694 - accuracy: 0.9251 - val_loss: 0.2722 - val_accuracy: 0.9245\n",
      "Epoch 137/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2692 - accuracy: 0.9247 - val_loss: 0.2718 - val_accuracy: 0.9250\n",
      "Epoch 138/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2691 - accuracy: 0.9249 - val_loss: 0.2719 - val_accuracy: 0.9250\n",
      "Epoch 139/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2689 - accuracy: 0.9253 - val_loss: 0.2719 - val_accuracy: 0.9252\n",
      "Epoch 140/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2688 - accuracy: 0.9256 - val_loss: 0.2716 - val_accuracy: 0.9254\n",
      "Epoch 141/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2686 - accuracy: 0.9250 - val_loss: 0.2715 - val_accuracy: 0.9252\n",
      "Epoch 142/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2684 - accuracy: 0.9256 - val_loss: 0.2716 - val_accuracy: 0.9252\n",
      "Epoch 143/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2683 - accuracy: 0.9254 - val_loss: 0.2715 - val_accuracy: 0.9246\n",
      "Epoch 144/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2682 - accuracy: 0.9251 - val_loss: 0.2714 - val_accuracy: 0.9247\n",
      "Epoch 145/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2680 - accuracy: 0.9255 - val_loss: 0.2712 - val_accuracy: 0.9249\n",
      "Epoch 146/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2678 - accuracy: 0.9251 - val_loss: 0.2711 - val_accuracy: 0.9248\n",
      "Epoch 147/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2677 - accuracy: 0.9257 - val_loss: 0.2712 - val_accuracy: 0.9249\n",
      "Epoch 148/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2676 - accuracy: 0.9256 - val_loss: 0.2709 - val_accuracy: 0.9249\n",
      "Epoch 149/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2674 - accuracy: 0.9256 - val_loss: 0.2710 - val_accuracy: 0.9249\n",
      "Epoch 150/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2673 - accuracy: 0.9260 - val_loss: 0.2710 - val_accuracy: 0.9247\n",
      "Epoch 151/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2672 - accuracy: 0.9256 - val_loss: 0.2708 - val_accuracy: 0.9248\n",
      "Epoch 152/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2670 - accuracy: 0.9255 - val_loss: 0.2709 - val_accuracy: 0.9250\n",
      "Epoch 153/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2669 - accuracy: 0.9258 - val_loss: 0.2707 - val_accuracy: 0.9250\n",
      "Epoch 154/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2668 - accuracy: 0.9257 - val_loss: 0.2706 - val_accuracy: 0.9252\n",
      "Epoch 155/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2666 - accuracy: 0.9256 - val_loss: 0.2706 - val_accuracy: 0.9248\n",
      "Epoch 156/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2665 - accuracy: 0.9258 - val_loss: 0.2705 - val_accuracy: 0.9256\n",
      "Epoch 157/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2663 - accuracy: 0.9257 - val_loss: 0.2704 - val_accuracy: 0.9252\n",
      "Epoch 158/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2662 - accuracy: 0.9258 - val_loss: 0.2704 - val_accuracy: 0.9251\n",
      "Epoch 159/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2661 - accuracy: 0.9258 - val_loss: 0.2704 - val_accuracy: 0.9249\n",
      "Epoch 160/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2660 - accuracy: 0.9258 - val_loss: 0.2704 - val_accuracy: 0.9252\n",
      "Epoch 161/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2658 - accuracy: 0.9263 - val_loss: 0.2704 - val_accuracy: 0.9256\n",
      "Epoch 162/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2657 - accuracy: 0.9259 - val_loss: 0.2703 - val_accuracy: 0.9250\n",
      "Epoch 163/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2656 - accuracy: 0.9259 - val_loss: 0.2701 - val_accuracy: 0.9247\n",
      "Epoch 164/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2655 - accuracy: 0.9262 - val_loss: 0.2700 - val_accuracy: 0.9249\n",
      "Epoch 165/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2653 - accuracy: 0.9260 - val_loss: 0.2700 - val_accuracy: 0.9252\n",
      "Epoch 166/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2652 - accuracy: 0.9262 - val_loss: 0.2700 - val_accuracy: 0.9253\n",
      "Epoch 167/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2651 - accuracy: 0.9262 - val_loss: 0.2698 - val_accuracy: 0.9249\n",
      "Epoch 168/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2650 - accuracy: 0.9263 - val_loss: 0.2699 - val_accuracy: 0.9252\n",
      "Epoch 169/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2648 - accuracy: 0.9264 - val_loss: 0.2699 - val_accuracy: 0.9254\n",
      "Epoch 170/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2647 - accuracy: 0.9264 - val_loss: 0.2697 - val_accuracy: 0.9256\n",
      "Epoch 171/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2646 - accuracy: 0.9262 - val_loss: 0.2697 - val_accuracy: 0.9256\n",
      "Epoch 172/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2645 - accuracy: 0.9264 - val_loss: 0.2696 - val_accuracy: 0.9256\n",
      "Epoch 173/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2644 - accuracy: 0.9263 - val_loss: 0.2695 - val_accuracy: 0.9254\n",
      "Epoch 174/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2643 - accuracy: 0.9265 - val_loss: 0.2696 - val_accuracy: 0.9256\n",
      "Epoch 175/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2641 - accuracy: 0.9264 - val_loss: 0.2694 - val_accuracy: 0.9254\n",
      "Epoch 176/200\n",
      "47999/47999 [==============================] - 1s 25us/step - loss: 0.2640 - accuracy: 0.9266 - val_loss: 0.2693 - val_accuracy: 0.9252\n",
      "Epoch 177/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2639 - accuracy: 0.9264 - val_loss: 0.2695 - val_accuracy: 0.9257\n",
      "Epoch 178/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2638 - accuracy: 0.9266 - val_loss: 0.2693 - val_accuracy: 0.9252\n",
      "Epoch 179/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2636 - accuracy: 0.9263 - val_loss: 0.2693 - val_accuracy: 0.9260\n",
      "Epoch 180/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2635 - accuracy: 0.9269 - val_loss: 0.2692 - val_accuracy: 0.9253\n",
      "Epoch 181/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2634 - accuracy: 0.9267 - val_loss: 0.2693 - val_accuracy: 0.9253\n",
      "Epoch 182/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2633 - accuracy: 0.9265 - val_loss: 0.2692 - val_accuracy: 0.9257\n",
      "Epoch 183/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2632 - accuracy: 0.9267 - val_loss: 0.2690 - val_accuracy: 0.9257\n",
      "Epoch 184/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2631 - accuracy: 0.9265 - val_loss: 0.2691 - val_accuracy: 0.9252\n",
      "Epoch 185/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2630 - accuracy: 0.9267 - val_loss: 0.2692 - val_accuracy: 0.9257\n",
      "Epoch 186/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2629 - accuracy: 0.9269 - val_loss: 0.2689 - val_accuracy: 0.9261\n",
      "Epoch 187/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2628 - accuracy: 0.9269 - val_loss: 0.2691 - val_accuracy: 0.9254\n",
      "Epoch 188/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2627 - accuracy: 0.9268 - val_loss: 0.2688 - val_accuracy: 0.9259\n",
      "Epoch 189/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2625 - accuracy: 0.9267 - val_loss: 0.2689 - val_accuracy: 0.9256\n",
      "Epoch 190/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2625 - accuracy: 0.9268 - val_loss: 0.2686 - val_accuracy: 0.9255\n",
      "Epoch 191/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2624 - accuracy: 0.9269 - val_loss: 0.2687 - val_accuracy: 0.9258\n",
      "Epoch 192/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2623 - accuracy: 0.9272 - val_loss: 0.2688 - val_accuracy: 0.9260\n",
      "Epoch 193/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2622 - accuracy: 0.9269 - val_loss: 0.2687 - val_accuracy: 0.9259\n",
      "Epoch 194/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2621 - accuracy: 0.9267 - val_loss: 0.2685 - val_accuracy: 0.9265\n",
      "Epoch 195/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2620 - accuracy: 0.9271 - val_loss: 0.2687 - val_accuracy: 0.9259\n",
      "Epoch 196/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2619 - accuracy: 0.9269 - val_loss: 0.2686 - val_accuracy: 0.9261\n",
      "Epoch 197/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2617 - accuracy: 0.9270 - val_loss: 0.2687 - val_accuracy: 0.9258\n",
      "Epoch 198/200\n",
      "47999/47999 [==============================] - 1s 24us/step - loss: 0.2617 - accuracy: 0.9271 - val_loss: 0.2685 - val_accuracy: 0.9260\n",
      "Epoch 199/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2616 - accuracy: 0.9270 - val_loss: 0.2683 - val_accuracy: 0.9261\n",
      "Epoch 200/200\n",
      "47999/47999 [==============================] - 1s 23us/step - loss: 0.2614 - accuracy: 0.9272 - val_loss: 0.2684 - val_accuracy: 0.9262\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(xtrain, ytrain,\n",
    "batch_size=BATCH_SIZE, epochs=NB_EPOCH,\n",
    "verbose=VERBOSE, validation_split=VALIDATION_SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9999/9999 [==============================] - 0s 13us/step\n",
      "Test score: 0.271035835871918\n",
      "Test accuracy: 0.9233923554420471\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(xtest, ytest, verbose=VERBOSE)\n",
    "print(\"Test score:\", score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nueral Network with Keras Using MNIST data with two hidden layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_10 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 10 outputs\n",
    "# final stage is softmax\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(NB_CLASSES, input_shape=(N_HIDDEN,)))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 47999 samples, validate on 12000 samples\n",
      "Epoch 1/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.9430 - accuracy: 0.7779 - val_loss: 0.4857 - val_accuracy: 0.8815\n",
      "Epoch 2/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.4484 - accuracy: 0.8815 - val_loss: 0.3724 - val_accuracy: 0.9013\n",
      "Epoch 3/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.3750 - accuracy: 0.8962 - val_loss: 0.3302 - val_accuracy: 0.9087\n",
      "Epoch 4/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.3389 - accuracy: 0.9047 - val_loss: 0.3051 - val_accuracy: 0.9157\n",
      "Epoch 5/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.3149 - accuracy: 0.9111 - val_loss: 0.2884 - val_accuracy: 0.9188\n",
      "Epoch 6/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2969 - accuracy: 0.9164 - val_loss: 0.2747 - val_accuracy: 0.9244\n",
      "Epoch 7/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2817 - accuracy: 0.9210 - val_loss: 0.2632 - val_accuracy: 0.9258\n",
      "Epoch 8/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.2690 - accuracy: 0.9244 - val_loss: 0.2527 - val_accuracy: 0.9297\n",
      "Epoch 9/200\n",
      "47999/47999 [==============================] - 2s 36us/step - loss: 0.2576 - accuracy: 0.9282 - val_loss: 0.2432 - val_accuracy: 0.9321\n",
      "Epoch 10/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.2473 - accuracy: 0.9306 - val_loss: 0.2351 - val_accuracy: 0.9349\n",
      "Epoch 11/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.2382 - accuracy: 0.9333 - val_loss: 0.2266 - val_accuracy: 0.9374\n",
      "Epoch 12/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2299 - accuracy: 0.9359 - val_loss: 0.2199 - val_accuracy: 0.9391\n",
      "Epoch 13/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2222 - accuracy: 0.9375 - val_loss: 0.2136 - val_accuracy: 0.9411\n",
      "Epoch 14/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2150 - accuracy: 0.9394 - val_loss: 0.2077 - val_accuracy: 0.9433\n",
      "Epoch 15/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.2083 - accuracy: 0.9415 - val_loss: 0.2035 - val_accuracy: 0.9450\n",
      "Epoch 16/200\n",
      "47999/47999 [==============================] - 2s 37us/step - loss: 0.2021 - accuracy: 0.9435 - val_loss: 0.1974 - val_accuracy: 0.9458\n",
      "Epoch 17/200\n",
      "47999/47999 [==============================] - 2s 34us/step - loss: 0.1962 - accuracy: 0.9446 - val_loss: 0.1925 - val_accuracy: 0.9485\n",
      "Epoch 18/200\n",
      "47999/47999 [==============================] - 2s 38us/step - loss: 0.1907 - accuracy: 0.9461 - val_loss: 0.1883 - val_accuracy: 0.9490\n",
      "Epoch 19/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1855 - accuracy: 0.9473 - val_loss: 0.1842 - val_accuracy: 0.9502\n",
      "Epoch 20/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1804 - accuracy: 0.9488 - val_loss: 0.1809 - val_accuracy: 0.9509\n",
      "Epoch 21/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1760 - accuracy: 0.9500 - val_loss: 0.1763 - val_accuracy: 0.9515\n",
      "Epoch 22/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1714 - accuracy: 0.9514 - val_loss: 0.1726 - val_accuracy: 0.9529\n",
      "Epoch 23/200\n",
      "47999/47999 [==============================] - 2s 33us/step - loss: 0.1673 - accuracy: 0.9524 - val_loss: 0.1696 - val_accuracy: 0.9532\n",
      "Epoch 24/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1632 - accuracy: 0.9538 - val_loss: 0.1660 - val_accuracy: 0.9546\n",
      "Epoch 25/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1592 - accuracy: 0.9551 - val_loss: 0.1640 - val_accuracy: 0.9541\n",
      "Epoch 26/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1556 - accuracy: 0.9565 - val_loss: 0.1606 - val_accuracy: 0.9558\n",
      "Epoch 27/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1521 - accuracy: 0.9571 - val_loss: 0.1584 - val_accuracy: 0.9553\n",
      "Epoch 28/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1488 - accuracy: 0.9578 - val_loss: 0.1549 - val_accuracy: 0.9567\n",
      "Epoch 29/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1455 - accuracy: 0.9589 - val_loss: 0.1527 - val_accuracy: 0.9578\n",
      "Epoch 30/200\n",
      "47999/47999 [==============================] - 2s 35us/step - loss: 0.1423 - accuracy: 0.9597 - val_loss: 0.1499 - val_accuracy: 0.9588\n",
      "Epoch 31/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1393 - accuracy: 0.9612 - val_loss: 0.1478 - val_accuracy: 0.9592\n",
      "Epoch 32/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1364 - accuracy: 0.9620 - val_loss: 0.1464 - val_accuracy: 0.9595\n",
      "Epoch 33/200\n",
      "47999/47999 [==============================] - 2s 33us/step - loss: 0.1337 - accuracy: 0.9633 - val_loss: 0.1436 - val_accuracy: 0.9608\n",
      "Epoch 34/200\n",
      "47999/47999 [==============================] - 2s 36us/step - loss: 0.1309 - accuracy: 0.9640 - val_loss: 0.1424 - val_accuracy: 0.9615\n",
      "Epoch 35/200\n",
      "47999/47999 [==============================] - 2s 38us/step - loss: 0.1285 - accuracy: 0.9647 - val_loss: 0.1401 - val_accuracy: 0.9622\n",
      "Epoch 36/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1260 - accuracy: 0.9653 - val_loss: 0.1383 - val_accuracy: 0.9618\n",
      "Epoch 37/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1235 - accuracy: 0.9664 - val_loss: 0.1362 - val_accuracy: 0.9633\n",
      "Epoch 38/200\n",
      "47999/47999 [==============================] - 2s 39us/step - loss: 0.1213 - accuracy: 0.9669 - val_loss: 0.1344 - val_accuracy: 0.9634\n",
      "Epoch 39/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1189 - accuracy: 0.9677 - val_loss: 0.1333 - val_accuracy: 0.9638\n",
      "Epoch 40/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1169 - accuracy: 0.9683 - val_loss: 0.1315 - val_accuracy: 0.9638\n",
      "Epoch 41/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1147 - accuracy: 0.9692 - val_loss: 0.1302 - val_accuracy: 0.9641\n",
      "Epoch 42/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1129 - accuracy: 0.9696 - val_loss: 0.1290 - val_accuracy: 0.9648\n",
      "Epoch 43/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.1109 - accuracy: 0.9700 - val_loss: 0.1278 - val_accuracy: 0.9653\n",
      "Epoch 44/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.1090 - accuracy: 0.9710 - val_loss: 0.1261 - val_accuracy: 0.9650\n",
      "Epoch 45/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1071 - accuracy: 0.9714 - val_loss: 0.1248 - val_accuracy: 0.9653\n",
      "Epoch 46/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1053 - accuracy: 0.9717 - val_loss: 0.1246 - val_accuracy: 0.9656\n",
      "Epoch 47/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1037 - accuracy: 0.9721 - val_loss: 0.1228 - val_accuracy: 0.9659\n",
      "Epoch 48/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1021 - accuracy: 0.9727 - val_loss: 0.1220 - val_accuracy: 0.9664\n",
      "Epoch 49/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.1004 - accuracy: 0.9734 - val_loss: 0.1220 - val_accuracy: 0.9668\n",
      "Epoch 50/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0989 - accuracy: 0.9740 - val_loss: 0.1195 - val_accuracy: 0.9669\n",
      "Epoch 51/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0973 - accuracy: 0.9739 - val_loss: 0.1194 - val_accuracy: 0.9664\n",
      "Epoch 52/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0960 - accuracy: 0.9744 - val_loss: 0.1177 - val_accuracy: 0.9673\n",
      "Epoch 53/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0945 - accuracy: 0.9748 - val_loss: 0.1172 - val_accuracy: 0.9671\n",
      "Epoch 54/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0931 - accuracy: 0.9756 - val_loss: 0.1165 - val_accuracy: 0.9670\n",
      "Epoch 55/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0917 - accuracy: 0.9759 - val_loss: 0.1149 - val_accuracy: 0.9670\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0904 - accuracy: 0.9764 - val_loss: 0.1143 - val_accuracy: 0.9677\n",
      "Epoch 57/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0892 - accuracy: 0.9763 - val_loss: 0.1134 - val_accuracy: 0.9675\n",
      "Epoch 58/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0879 - accuracy: 0.9771 - val_loss: 0.1128 - val_accuracy: 0.9680\n",
      "Epoch 59/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0867 - accuracy: 0.9771 - val_loss: 0.1119 - val_accuracy: 0.9676\n",
      "Epoch 60/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0855 - accuracy: 0.9773 - val_loss: 0.1115 - val_accuracy: 0.9678\n",
      "Epoch 61/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0844 - accuracy: 0.9778 - val_loss: 0.1109 - val_accuracy: 0.9681\n",
      "Epoch 62/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0833 - accuracy: 0.9782 - val_loss: 0.1100 - val_accuracy: 0.9689\n",
      "Epoch 63/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0822 - accuracy: 0.9788 - val_loss: 0.1090 - val_accuracy: 0.9678\n",
      "Epoch 64/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0810 - accuracy: 0.9786 - val_loss: 0.1082 - val_accuracy: 0.9687\n",
      "Epoch 65/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0800 - accuracy: 0.9790 - val_loss: 0.1078 - val_accuracy: 0.9688\n",
      "Epoch 66/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0789 - accuracy: 0.9796 - val_loss: 0.1079 - val_accuracy: 0.9696\n",
      "Epoch 67/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0778 - accuracy: 0.9799 - val_loss: 0.1067 - val_accuracy: 0.9688\n",
      "Epoch 68/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0770 - accuracy: 0.9801 - val_loss: 0.1060 - val_accuracy: 0.9698\n",
      "Epoch 69/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0760 - accuracy: 0.9802 - val_loss: 0.1058 - val_accuracy: 0.9695\n",
      "Epoch 70/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0750 - accuracy: 0.9809 - val_loss: 0.1055 - val_accuracy: 0.9702\n",
      "Epoch 71/200\n",
      "47999/47999 [==============================] - 2s 37us/step - loss: 0.0742 - accuracy: 0.9808 - val_loss: 0.1049 - val_accuracy: 0.9696\n",
      "Epoch 72/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0733 - accuracy: 0.9811 - val_loss: 0.1039 - val_accuracy: 0.9705\n",
      "Epoch 73/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0723 - accuracy: 0.9814 - val_loss: 0.1036 - val_accuracy: 0.9707\n",
      "Epoch 74/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0716 - accuracy: 0.9816 - val_loss: 0.1032 - val_accuracy: 0.9709\n",
      "Epoch 75/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0706 - accuracy: 0.9821 - val_loss: 0.1028 - val_accuracy: 0.9708\n",
      "Epoch 76/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0698 - accuracy: 0.9826 - val_loss: 0.1022 - val_accuracy: 0.9707\n",
      "Epoch 77/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0690 - accuracy: 0.9825 - val_loss: 0.1018 - val_accuracy: 0.9709\n",
      "Epoch 78/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0682 - accuracy: 0.9827 - val_loss: 0.1010 - val_accuracy: 0.9709\n",
      "Epoch 79/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0674 - accuracy: 0.9830 - val_loss: 0.1009 - val_accuracy: 0.9711\n",
      "Epoch 80/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0666 - accuracy: 0.9832 - val_loss: 0.1003 - val_accuracy: 0.9714\n",
      "Epoch 81/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0659 - accuracy: 0.9834 - val_loss: 0.1004 - val_accuracy: 0.9712\n",
      "Epoch 82/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0651 - accuracy: 0.9837 - val_loss: 0.0995 - val_accuracy: 0.9718\n",
      "Epoch 83/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0644 - accuracy: 0.9840 - val_loss: 0.0989 - val_accuracy: 0.9716\n",
      "Epoch 84/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0637 - accuracy: 0.9842 - val_loss: 0.0989 - val_accuracy: 0.9720\n",
      "Epoch 85/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0631 - accuracy: 0.9843 - val_loss: 0.0982 - val_accuracy: 0.9716\n",
      "Epoch 86/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0622 - accuracy: 0.9844 - val_loss: 0.0979 - val_accuracy: 0.9724\n",
      "Epoch 87/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0616 - accuracy: 0.9844 - val_loss: 0.0981 - val_accuracy: 0.9715\n",
      "Epoch 88/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0609 - accuracy: 0.9847 - val_loss: 0.0980 - val_accuracy: 0.9722\n",
      "Epoch 89/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0603 - accuracy: 0.9852 - val_loss: 0.0969 - val_accuracy: 0.9726\n",
      "Epoch 90/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0596 - accuracy: 0.9852 - val_loss: 0.0970 - val_accuracy: 0.9722\n",
      "Epoch 91/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0590 - accuracy: 0.9854 - val_loss: 0.0964 - val_accuracy: 0.9722\n",
      "Epoch 92/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0583 - accuracy: 0.9856 - val_loss: 0.0967 - val_accuracy: 0.9727\n",
      "Epoch 93/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0578 - accuracy: 0.9858 - val_loss: 0.0961 - val_accuracy: 0.9729\n",
      "Epoch 94/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0572 - accuracy: 0.9859 - val_loss: 0.0954 - val_accuracy: 0.9734\n",
      "Epoch 95/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0566 - accuracy: 0.9861 - val_loss: 0.0949 - val_accuracy: 0.9730\n",
      "Epoch 96/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0559 - accuracy: 0.9861 - val_loss: 0.0949 - val_accuracy: 0.9722\n",
      "Epoch 97/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0554 - accuracy: 0.9866 - val_loss: 0.0947 - val_accuracy: 0.9728\n",
      "Epoch 98/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0549 - accuracy: 0.9864 - val_loss: 0.0948 - val_accuracy: 0.9728\n",
      "Epoch 99/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0544 - accuracy: 0.9866 - val_loss: 0.0943 - val_accuracy: 0.9730\n",
      "Epoch 100/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0538 - accuracy: 0.9869 - val_loss: 0.0943 - val_accuracy: 0.9727\n",
      "Epoch 101/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0532 - accuracy: 0.9869 - val_loss: 0.0934 - val_accuracy: 0.9730\n",
      "Epoch 102/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0527 - accuracy: 0.9872 - val_loss: 0.0938 - val_accuracy: 0.9728\n",
      "Epoch 103/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0522 - accuracy: 0.9874 - val_loss: 0.0931 - val_accuracy: 0.9735\n",
      "Epoch 104/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0516 - accuracy: 0.9872 - val_loss: 0.0931 - val_accuracy: 0.9730\n",
      "Epoch 105/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0512 - accuracy: 0.9872 - val_loss: 0.0925 - val_accuracy: 0.9733\n",
      "Epoch 106/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0507 - accuracy: 0.9876 - val_loss: 0.0924 - val_accuracy: 0.9737\n",
      "Epoch 107/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0502 - accuracy: 0.9879 - val_loss: 0.0921 - val_accuracy: 0.9732\n",
      "Epoch 108/200\n",
      "47999/47999 [==============================] - 1s 29us/step - loss: 0.0497 - accuracy: 0.9880 - val_loss: 0.0924 - val_accuracy: 0.9728\n",
      "Epoch 109/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0492 - accuracy: 0.9880 - val_loss: 0.0921 - val_accuracy: 0.9737\n",
      "Epoch 110/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0487 - accuracy: 0.9881 - val_loss: 0.0921 - val_accuracy: 0.9737\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 111/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0484 - accuracy: 0.9886 - val_loss: 0.0916 - val_accuracy: 0.9728\n",
      "Epoch 112/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0478 - accuracy: 0.9886 - val_loss: 0.0914 - val_accuracy: 0.9732\n",
      "Epoch 113/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0474 - accuracy: 0.9885 - val_loss: 0.0908 - val_accuracy: 0.9737\n",
      "Epoch 114/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0469 - accuracy: 0.9887 - val_loss: 0.0907 - val_accuracy: 0.9737\n",
      "Epoch 115/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0465 - accuracy: 0.9890 - val_loss: 0.0909 - val_accuracy: 0.9743\n",
      "Epoch 116/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0459 - accuracy: 0.9893 - val_loss: 0.0904 - val_accuracy: 0.9741\n",
      "Epoch 117/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0457 - accuracy: 0.9892 - val_loss: 0.0905 - val_accuracy: 0.9739\n",
      "Epoch 118/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0451 - accuracy: 0.9895 - val_loss: 0.0914 - val_accuracy: 0.9735\n",
      "Epoch 119/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0448 - accuracy: 0.9894 - val_loss: 0.0908 - val_accuracy: 0.9738\n",
      "Epoch 120/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0444 - accuracy: 0.9897 - val_loss: 0.0899 - val_accuracy: 0.9740\n",
      "Epoch 121/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0440 - accuracy: 0.9899 - val_loss: 0.0897 - val_accuracy: 0.9742\n",
      "Epoch 122/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0436 - accuracy: 0.9899 - val_loss: 0.0901 - val_accuracy: 0.9740\n",
      "Epoch 123/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0431 - accuracy: 0.9900 - val_loss: 0.0893 - val_accuracy: 0.9744\n",
      "Epoch 124/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0427 - accuracy: 0.9901 - val_loss: 0.0897 - val_accuracy: 0.9743\n",
      "Epoch 125/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0424 - accuracy: 0.9902 - val_loss: 0.0892 - val_accuracy: 0.9741\n",
      "Epoch 126/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0420 - accuracy: 0.9903 - val_loss: 0.0890 - val_accuracy: 0.9743\n",
      "Epoch 127/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0417 - accuracy: 0.9905 - val_loss: 0.0890 - val_accuracy: 0.9742\n",
      "Epoch 128/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0413 - accuracy: 0.9906 - val_loss: 0.0893 - val_accuracy: 0.9741\n",
      "Epoch 129/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0409 - accuracy: 0.9908 - val_loss: 0.0888 - val_accuracy: 0.9737\n",
      "Epoch 130/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0406 - accuracy: 0.9907 - val_loss: 0.0887 - val_accuracy: 0.9747\n",
      "Epoch 131/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0402 - accuracy: 0.9911 - val_loss: 0.0883 - val_accuracy: 0.9742\n",
      "Epoch 132/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0398 - accuracy: 0.9911 - val_loss: 0.0881 - val_accuracy: 0.9744\n",
      "Epoch 133/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0395 - accuracy: 0.9909 - val_loss: 0.0879 - val_accuracy: 0.9744\n",
      "Epoch 134/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0392 - accuracy: 0.9912 - val_loss: 0.0882 - val_accuracy: 0.9743\n",
      "Epoch 135/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0388 - accuracy: 0.9912 - val_loss: 0.0879 - val_accuracy: 0.9746\n",
      "Epoch 136/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0385 - accuracy: 0.9916 - val_loss: 0.0878 - val_accuracy: 0.9745\n",
      "Epoch 137/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0382 - accuracy: 0.9916 - val_loss: 0.0882 - val_accuracy: 0.9737\n",
      "Epoch 138/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0378 - accuracy: 0.9917 - val_loss: 0.0878 - val_accuracy: 0.9739\n",
      "Epoch 139/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0375 - accuracy: 0.9916 - val_loss: 0.0875 - val_accuracy: 0.9743\n",
      "Epoch 140/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0372 - accuracy: 0.9919 - val_loss: 0.0876 - val_accuracy: 0.9748\n",
      "Epoch 141/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0369 - accuracy: 0.9921 - val_loss: 0.0877 - val_accuracy: 0.9743\n",
      "Epoch 142/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0365 - accuracy: 0.9919 - val_loss: 0.0875 - val_accuracy: 0.9746\n",
      "Epoch 143/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0363 - accuracy: 0.9921 - val_loss: 0.0873 - val_accuracy: 0.9747\n",
      "Epoch 144/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0359 - accuracy: 0.9921 - val_loss: 0.0872 - val_accuracy: 0.9738\n",
      "Epoch 145/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0356 - accuracy: 0.9922 - val_loss: 0.0870 - val_accuracy: 0.9745\n",
      "Epoch 146/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0354 - accuracy: 0.9922 - val_loss: 0.0867 - val_accuracy: 0.9748\n",
      "Epoch 147/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0350 - accuracy: 0.9923 - val_loss: 0.0869 - val_accuracy: 0.9749\n",
      "Epoch 148/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0347 - accuracy: 0.9927 - val_loss: 0.0866 - val_accuracy: 0.9746\n",
      "Epoch 149/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0344 - accuracy: 0.9926 - val_loss: 0.0869 - val_accuracy: 0.9740\n",
      "Epoch 150/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0342 - accuracy: 0.9926 - val_loss: 0.0869 - val_accuracy: 0.9746\n",
      "Epoch 151/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0339 - accuracy: 0.9928 - val_loss: 0.0866 - val_accuracy: 0.9750\n",
      "Epoch 152/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0335 - accuracy: 0.9929 - val_loss: 0.0867 - val_accuracy: 0.9748\n",
      "Epoch 153/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0333 - accuracy: 0.9929 - val_loss: 0.0866 - val_accuracy: 0.9746\n",
      "Epoch 154/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0330 - accuracy: 0.9928 - val_loss: 0.0866 - val_accuracy: 0.9747\n",
      "Epoch 155/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0328 - accuracy: 0.9930 - val_loss: 0.0863 - val_accuracy: 0.9747\n",
      "Epoch 156/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0325 - accuracy: 0.9933 - val_loss: 0.0864 - val_accuracy: 0.9749\n",
      "Epoch 157/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0322 - accuracy: 0.9932 - val_loss: 0.0859 - val_accuracy: 0.9746\n",
      "Epoch 158/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0320 - accuracy: 0.9933 - val_loss: 0.0860 - val_accuracy: 0.9753\n",
      "Epoch 159/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0317 - accuracy: 0.9933 - val_loss: 0.0858 - val_accuracy: 0.9745\n",
      "Epoch 160/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0314 - accuracy: 0.9935 - val_loss: 0.0859 - val_accuracy: 0.9747\n",
      "Epoch 161/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0312 - accuracy: 0.9935 - val_loss: 0.0857 - val_accuracy: 0.9749\n",
      "Epoch 162/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0309 - accuracy: 0.9936 - val_loss: 0.0863 - val_accuracy: 0.9747\n",
      "Epoch 163/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0306 - accuracy: 0.9938 - val_loss: 0.0865 - val_accuracy: 0.9743\n",
      "Epoch 164/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0305 - accuracy: 0.9938 - val_loss: 0.0860 - val_accuracy: 0.9751\n",
      "Epoch 165/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0302 - accuracy: 0.9936 - val_loss: 0.0854 - val_accuracy: 0.9749\n",
      "Epoch 166/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0300 - accuracy: 0.9941 - val_loss: 0.0856 - val_accuracy: 0.9753\n",
      "Epoch 167/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0298 - accuracy: 0.9939 - val_loss: 0.0857 - val_accuracy: 0.9747\n",
      "Epoch 168/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0295 - accuracy: 0.9941 - val_loss: 0.0856 - val_accuracy: 0.9753\n",
      "Epoch 169/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0292 - accuracy: 0.9941 - val_loss: 0.0857 - val_accuracy: 0.9750\n",
      "Epoch 170/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0290 - accuracy: 0.9943 - val_loss: 0.0855 - val_accuracy: 0.9747\n",
      "Epoch 171/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0288 - accuracy: 0.9944 - val_loss: 0.0855 - val_accuracy: 0.9749\n",
      "Epoch 172/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0286 - accuracy: 0.9942 - val_loss: 0.0851 - val_accuracy: 0.9751\n",
      "Epoch 173/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0284 - accuracy: 0.9944 - val_loss: 0.0856 - val_accuracy: 0.9747\n",
      "Epoch 174/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0281 - accuracy: 0.9945 - val_loss: 0.0855 - val_accuracy: 0.9753\n",
      "Epoch 175/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0279 - accuracy: 0.9947 - val_loss: 0.0852 - val_accuracy: 0.9747\n",
      "Epoch 176/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0277 - accuracy: 0.9947 - val_loss: 0.0857 - val_accuracy: 0.9747\n",
      "Epoch 177/200\n",
      "47999/47999 [==============================] - 2s 31us/step - loss: 0.0275 - accuracy: 0.9948 - val_loss: 0.0857 - val_accuracy: 0.9748\n",
      "Epoch 178/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0272 - accuracy: 0.9950 - val_loss: 0.0853 - val_accuracy: 0.9751\n",
      "Epoch 179/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0271 - accuracy: 0.9949 - val_loss: 0.0849 - val_accuracy: 0.9747\n",
      "Epoch 180/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0269 - accuracy: 0.9950 - val_loss: 0.0850 - val_accuracy: 0.9751\n",
      "Epoch 181/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0266 - accuracy: 0.9950 - val_loss: 0.0848 - val_accuracy: 0.9749\n",
      "Epoch 182/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0264 - accuracy: 0.9953 - val_loss: 0.0850 - val_accuracy: 0.9748\n",
      "Epoch 183/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0262 - accuracy: 0.9951 - val_loss: 0.0851 - val_accuracy: 0.9749\n",
      "Epoch 184/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0260 - accuracy: 0.9952 - val_loss: 0.0850 - val_accuracy: 0.9749\n",
      "Epoch 185/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0259 - accuracy: 0.9953 - val_loss: 0.0849 - val_accuracy: 0.9751\n",
      "Epoch 186/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0256 - accuracy: 0.9955 - val_loss: 0.0849 - val_accuracy: 0.9754\n",
      "Epoch 187/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0254 - accuracy: 0.9952 - val_loss: 0.0849 - val_accuracy: 0.9749\n",
      "Epoch 188/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0252 - accuracy: 0.9956 - val_loss: 0.0845 - val_accuracy: 0.9748\n",
      "Epoch 189/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0251 - accuracy: 0.9957 - val_loss: 0.0848 - val_accuracy: 0.9747\n",
      "Epoch 190/200\n",
      "47999/47999 [==============================] - 1s 29us/step - loss: 0.0249 - accuracy: 0.9956 - val_loss: 0.0849 - val_accuracy: 0.9751\n",
      "Epoch 191/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0247 - accuracy: 0.9957 - val_loss: 0.0847 - val_accuracy: 0.9745\n",
      "Epoch 192/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0245 - accuracy: 0.9957 - val_loss: 0.0849 - val_accuracy: 0.9751\n",
      "Epoch 193/200\n",
      "47999/47999 [==============================] - 1s 30us/step - loss: 0.0243 - accuracy: 0.9958 - val_loss: 0.0846 - val_accuracy: 0.9750\n",
      "Epoch 194/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0241 - accuracy: 0.9958 - val_loss: 0.0845 - val_accuracy: 0.9747\n",
      "Epoch 195/200\n",
      "47999/47999 [==============================] - 2s 35us/step - loss: 0.0240 - accuracy: 0.9959 - val_loss: 0.0851 - val_accuracy: 0.9744\n",
      "Epoch 196/200\n",
      "47999/47999 [==============================] - 2s 36us/step - loss: 0.0238 - accuracy: 0.9961 - val_loss: 0.0852 - val_accuracy: 0.9747\n",
      "Epoch 197/200\n",
      "47999/47999 [==============================] - 2s 32us/step - loss: 0.0236 - accuracy: 0.9959 - val_loss: 0.0848 - val_accuracy: 0.9749\n",
      "Epoch 198/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0234 - accuracy: 0.9960 - val_loss: 0.0848 - val_accuracy: 0.9751\n",
      "Epoch 199/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0233 - accuracy: 0.9961 - val_loss: 0.0847 - val_accuracy: 0.9749\n",
      "Epoch 200/200\n",
      "47999/47999 [==============================] - 1s 31us/step - loss: 0.0231 - accuracy: 0.9960 - val_loss: 0.0850 - val_accuracy: 0.9744\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(xtrain, ytrain,\n",
    "batch_size=BATCH_SIZE, epochs=NB_EPOCH,\n",
    "verbose=VERBOSE, validation_split=VALIDATION_SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9999/9999 [==============================] - 0s 16us/step\n",
      "Test score: 0.0742178417334397\n",
      "Test accuracy: 0.9774977564811707\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(xtest, ytest, verbose=VERBOSE)\n",
    "print(\"Test score:\", score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}