workshop_3_dnn_implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create anaconda environment\n",
    "<br>\n",
    "```bash\n",
    "conda create -n ml python=3.7.5 jupyter\n",
    "```\n",
    "Install fastai library\n",
    "<br>\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y_str = fetch_openml('mnist_784', version=1, return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1a1cf30fd0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAMBUlEQVR4nO3db4wcdR3H8c+HchQsaFpKawMVECFIjBY8q0n9gyFiIZpCAkoTSTXEYgIKhqgEHsATE6L8kQcEckilGISQAKEPEGkqCZIo4cBSClUKWGhp0wNrwv9ybb8+uMEc5XZ2uzO7s/J9v5LN7s5v9uaT7X06uzuz93NECMCH335NBwDQH5QdSIKyA0lQdiAJyg4ksX8/N3aAp8eBmtHPTQKpvKM39W7s9FRjlcpue7Gk6yVNk/TbiLiqbP0DNUNf9ClVNgmgxKOxpuVY1y/jbU+TdIOk0ySdIGmp7RO6/XkAeqvKe/aFkp6LiBci4l1Jd0paUk8sAHWrUvbDJW2edH9Lsex9bC+3PWp7dFw7K2wOQBVVyj7VhwAfOPc2IkYiYjgihoc0vcLmAFRRpexbJM2fdP8ISVurxQHQK1XK/pikY20fbfsASedIWlVPLAB16/rQW0Tssn2hpD9p4tDbioh4urZkAGpV6Th7RNwv6f6asgDoIU6XBZKg7EASlB1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiCsgNJUHYgCcoOJEHZgSQoO5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kQdmBJCg7kARlB5Kg7EASlB1IgrIDSVB2IIlKs7hi8Hn69NLxt077XOn4Zy9/snR84xd27nMmNKNS2W1vkvS6pN2SdkXEcB2hANSvjj371yPi1Rp+DoAe4j07kETVsoekB20/bnv5VCvYXm571PbouHh/BzSl6sv4RRGx1fYcSatt/yMiHp68QkSMSBqRpI96VlTcHoAuVdqzR8TW4npM0r2SFtYRCkD9ui677Rm2D3nvtqRTJa2vKxiAelV5GT9X0r223/s5f4iIB2pJhdpMO2x26fhDN9xUOv6Xd8p/RX599LdLx3f968XScfRP12WPiBcklZ+RAWBgcOgNSIKyA0lQdiAJyg4kQdmBJPiKK0p95cBdpeO//MSs0vH9OPQ2MNizA0lQdiAJyg4kQdmBJCg7kARlB5Kg7EASHGdHqWlmf/Bhwb8kkARlB5Kg7EASlB1IgrIDSVB2IAnKDiTBcXaU2h17SsfHP1L+K1Q+YTT6iT07kARlB5Kg7EASlB1IgrIDSVB2IAnKDiTBcXZUMvb5odLx+X/sUxC01XbPbnuF7THb6yctm2V7te2NxfXM3sYEUFUnL+NvlbR4r2WXSloTEcdKWlPcBzDA2pY9Ih6WtGOvxUskrSxur5R0Rs25ANSs2w/o5kbENkkqrue0WtH2ctujtkfHtbPLzQGoquefxkfESEQMR8TwEF+LABrTbdm3254nScX1WH2RAPRCt2VfJWlZcXuZpPvqiQOgV9oeZ7d9h6STJc22vUXSFZKuknSX7fMkvSTp7F6GRPdifLx0/Nnxd0rHjxs6sHT87aPf3edMaEbbskfE0hZDp9ScBUAPcboskARlB5Kg7EASlB1IgrIDSfAV1w+53dvLz3f6yfPfLR1/4HhOofiwYM8OJEHZgSQoO5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kQdmBJCg7kARlB5Kg7EASlB1IgrIDSfB9dlRy8Ky3mo6ADrFnB5Kg7EASlB1IgrIDSVB2IAnKDiRB2YEkOM6OSu4+6ebS8R9rUZ+SoJ22e3bbK2yP2V4/admVtl+2vba4nN7bmACq6uRl/K2SFk+x/LqIWFBc7q83FoC6tS17RDwsaUcfsgDooSof0F1oe13xMn9mq5VsL7c9ant0XDsrbA5AFd2W/UZJx0haIGmbpGtarRgRIxExHBHDQ5re5eYAVNVV2SNie0Tsjog9km6WtLDeWADq1lXZbc+bdPdMSetbrQtgMLQ9zm77DkknS5pte4ukKySdbHuBpJC0SdL5PcyIHtr8yPzyFY7vTw70XtuyR8TSKRbf0oMsAHqI02WBJCg7kARlB5Kg7EASlB1Igq+4Jnfw5qj0+ENc/vhpJxzXcmz3M89W2jb2DXt2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiC4+zJ7ber2uOn2aXjew4aqrYB1IY9O5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kwXH25Gbe+tfS8Zt+fmTp+I8+9mLp+MafHtBy7FPfK30oasaeHUiCsgNJUHYgCcoOJEHZgSQoO5AEZQeS4Dg7Sl39t2+Wji8+5Tel48ed3/pvw+/pKhG61XbPbnu+7Ydsb7D9tO2LiuWzbK+2vbG4ntn7uAC61cnL+F2SLomIT0v6kqQLbJ8g6VJJayLiWElrivsABlTbskfEtoh4orj9uqQNkg6XtETSymK1lZLO6FVIANXt0wd0to+SdKKkRyXNjYht0sR/CJLmtHjMctujtkfHtbNaWgBd67jstg+WdLekiyPitU4fFxEjETEcEcNDmt5NRgA16Kjstoc0UfTbI+KeYvF22/OK8XmSxnoTEUAd2h56s21Jt0jaEBHXThpaJWmZpKuK6/t6khADbbfa/Cnpt9/pUxK008lx9kWSzpX0lO21xbLLNFHyu2yfJ+klSWf3JiKAOrQte0Q8IrX87/uUeuMA6BVOlwWSoOxAEpQdSIKyA0lQdiAJvuKKSo7Z/6DS8X//YGHLsUNvKf8z1qgXe3YgCcoOJEHZgSQoO5AEZQeSoOxAEpQdSILj7Cj1u6+tKB3/z563S8dnr3uj5Vh0lQjdYs8OJEHZgSQoO5AEZQeSoOxAEpQdSIKyA0lwnB2lfrbhrNLxs478e+n4fm+2nvJrd1eJ0C327EASlB1IgrIDSVB2IAnKDiRB2YEkKDuQRCfzs8+XdJukj0vaI2kkIq63faWkH0p6pVj1soi4v1dB0YxZ33q2dPzPmtHmJ5Q/Hv3TyUk1uyRdEhFP2D5E0uO2Vxdj10XE1b2LB6AunczPvk3StuL267Y3SDq818EA1Guf3rPbPkrSiZIeLRZdaHud7RW2Z7Z4zHLbo7ZHx9X61EkAvdVx2W0fLOluSRdHxGuSbpR0jKQFmtjzXzPV4yJiJCKGI2J4SNNriAygGx2V3faQJop+e0TcI0kRsT0idkfEHkk3S2o9gx+AxrUtu21LukXShoi4dtLyeZNWO1PS+vrjAahLJ5/GL5J0rqSnbK8tll0maantBZr4i8CbJJ3fk4QAatHJp/GPSPIUQxxTB/6PcAYdkARlB5Kg7EASlB1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiCsgNJUHYgCUdE/zZmvyLpxUmLZkt6tW8B9s2gZhvUXBLZulVntiMj4rCpBvpa9g9s3B6NiOHGApQY1GyDmksiW7f6lY2X8UASlB1IoumyjzS8/TKDmm1Qc0lk61ZfsjX6nh1A/zS9ZwfQJ5QdSKKRsttebPuftp+zfWkTGVqxvcn2U7bX2h5tOMsK22O2109aNsv2atsbi+sp59hrKNuVtl8unru1tk9vKNt82w/Z3mD7adsXFcsbfe5KcvXleev7e3bb0zQxafc3JG2R9JikpRHxTF+DtGB7k6ThiGj8BAzbX5X0hqTbIuIzxbJfSdoREVcV/1HOjIhfDEi2KyW90fQ03sVsRfMmTzMu6QxJ31eDz11Jru+oD89bE3v2hZKei4gXIuJdSXdKWtJAjoEXEQ9L2rHX4iWSVha3V2ril6XvWmQbCBGxLSKeKG6/Lum9acYbfe5KcvVFE2U/XNLmSfe3aLDmew9JD9p+3PbypsNMYW5EbJMmfnkkzWk4z97aTuPdT3tNMz4wz103059X1UTZp5pKapCO/y2KiJMknSbpguLlKjrT0TTe/TLFNOMDodvpz6tqouxbJM2fdP8ISVsbyDGliNhaXI9JuleDNxX19vdm0C2uxxrO8z+DNI33VNOMawCeuyanP2+i7I9JOtb20bYPkHSOpFUN5PgA2zOKD05ke4akUzV4U1GvkrSsuL1M0n0NZnmfQZnGu9U042r4uWt8+vOI6PtF0uma+ET+eUmXN5GhRa5PSnqyuDzddDZJd2jiZd24Jl4RnSfpUElrJG0srmcNULbfS3pK0jpNFGteQ9m+rIm3huskrS0upzf93JXk6svzxumyQBKcQQckQdmBJCg7kARlB5Kg7EASlB1IgrIDSfwXWI2V5YzPzakAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img1 = X[8].reshape(28, 28)\n",
    "plt.imshow(img1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_str[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_int = y_str.astype(int)\n",
    "\n",
    "def encode_idx(idx):\n",
    "    cd = np.zeros(10)\n",
    "    cd[idx] = 1\n",
    "    \n",
    "    return cd\n",
    "    \n",
    "y_v = np.array([encode_idx(i) for i in y_int])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 10)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_v[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_norm = X / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = X_norm[:60000], X[60000:]\n",
    "y_train, y_test = y_v[:60000], y_v[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1a1cfb4e50>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOYElEQVR4nO3dbYxc5XnG8euKbUwxJvHGseMQFxzjFAg0Jl0ZkBFQoVCCIgGKCLGiiFBapwlOQutKUFoVWtHKrRIiSimSKS6m4iWQgPAHmsSyECRqcFmoAROHN+MS4+0aswIDIfZ6fffDjqsFdp5dZs68eO//T1rNzLnnzLk1cPmcmeeceRwRAjD5faDTDQBoD8IOJEHYgSQIO5AEYQeSmNrOjR3i6XGoZrRzk0Aqv9Fb2ht7PFatqbDbPkfS9ZKmSPrXiFhVev6hmqGTfVYzmwRQsDE21K01fBhve4qkGyV9TtLxkpbZPr7R1wPQWs18Zl8i6fmI2BoReyXdJem8atoCULVmwn6kpF+Nery9tuwdbC+33We7b0h7mtgcgGY0E/axvgR4z7m3EbE6InojoneapjexOQDNaCbs2yXNH/X445J2NNcOgFZpJuyPSlpke4HtQyR9SdK6atoCULWGh94iYp/tFZJ+rJGhtzUR8XRlnQGoVFPj7BHxgKQHKuoFQAtxuiyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJNDWLK7qfp5b/E0/5yOyWbv+ZPz+6bm34sP3FdY9auLNYP+wbLtb/97pD6tYe7/1+cd1dw28V6yffs7JYP+bPHinWO6GpsNveJukNScOS9kVEbxVNAaheFXv234+IXRW8DoAW4jM7kESzYQ9JP7H9mO3lYz3B9nLbfbb7hrSnyc0BaFSzh/FLI2KH7TmS1tv+ZUQ8PPoJEbFa0mpJOsI90eT2ADSoqT17ROyo3e6UdJ+kJVU0BaB6DYfd9gzbMw/cl3S2pM1VNQagWs0cxs+VdJ/tA69zR0T8qJKuJpkpxy0q1mP6tGJ9xxkfKtbfPqX+mHDPB8vjxT/9dHm8uZP+49czi/V/+OdzivWNJ95Rt/bi0NvFdVcNfLZY/9hPD75PpA2HPSK2Svp0hb0AaCGG3oAkCDuQBGEHkiDsQBKEHUiCS1wrMHzmZ4r16269sVj/5LT6l2JOZkMxXKz/9Q1fLdanvlUe/jr1nhV1azNf3ldcd/qu8tDcYX0bi/VuxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnL0C05/ZUaw/9pv5xfonpw1U2U6lVvafUqxvfbP8U9S3LvxB3drr+8vj5HP/6T+L9VY6+C5gHR97diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IwhHtG1E8wj1xss9q2/a6xeAlpxbru88p/9zzlCcPL9af+MYN77unA67d9bvF+qNnlMfRh197vViPU+v/APG2bxVX1YJlT5SfgPfYGBu0OwbHnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMvvDxfrwq4PF+ot31B8rf/r0NcV1l/z9N4v1OTd27ppyvH9NjbPbXmN7p+3No5b12F5v+7na7awqGwZQvYkcxt8q6d2z3l8paUNELJK0ofYYQBcbN+wR8bCkdx9Hnidpbe3+WknnV9wXgIo1+gXd3Ijol6Ta7Zx6T7S93Haf7b4h7WlwcwCa1fJv4yNidUT0RkTvNE1v9eYA1NFo2Adsz5Ok2u3O6loC0AqNhn2dpItr9y+WdH817QBolXF/N972nZLOlDTb9nZJV0taJelu25dKeknSha1scrIb3vVqU+sP7W58fvdPffkXxforN00pv8D+8hzr6B7jhj0iltUpcXYMcBDhdFkgCcIOJEHYgSQIO5AEYQeSYMrmSeC4K56tW7vkxPKgyb8dtaFYP+PCy4r1md9/pFhH92DPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+CZSmTX7168cV131p3dvF+pXX3las/8UXLyjW478/WLc2/+9+XlxXbfyZ8wzYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEkzZnNzgH55arN9+9XeK9QVTD21425+6bUWxvujm/mJ939ZtDW97smpqymYAkwNhB5Ig7EAShB1IgrADSRB2IAnCDiTBODuKYuniYv2IVduL9Ts/8eOGt33sg39UrP/O39S/jl+Shp/b2vC2D1ZNjbPbXmN7p+3No5ZdY/tl25tqf+dW2TCA6k3kMP5WSeeMsfx7EbG49vdAtW0BqNq4YY+IhyUNtqEXAC3UzBd0K2w/WTvMn1XvSbaX2+6z3TekPU1sDkAzGg37TZIWSlosqV/Sd+s9MSJWR0RvRPRO0/QGNwegWQ2FPSIGImI4IvZLulnSkmrbAlC1hsJue96ohxdI2lzvuQC6w7jj7LbvlHSmpNmSBiRdXXu8WFJI2ibpaxFRvvhYjLNPRlPmzinWd1x0TN3axiuuL677gXH2RV9+8exi/fXTXi3WJ6PSOPu4k0RExLIxFt/SdFcA2orTZYEkCDuQBGEHkiDsQBKEHUiCS1zRMXdvL0/ZfJgPKdZ/HXuL9c9/8/L6r33fxuK6Byt+ShoAYQeyIOxAEoQdSIKwA0kQdiAJwg4kMe5Vb8ht/2nln5J+4cLylM0nLN5WtzbeOPp4bhg8qVg/7P6+pl5/smHPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+ybn3hGL92W+Vx7pvXrq2WD/90PI15c3YE0PF+iODC8ovsH/cXzdPhT07kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBOPtBYOqCo4r1Fy75WN3aNRfdVVz3C4fvaqinKlw10FusP3T9KcX6rLXl353HO427Z7c93/aDtrfYftr2t2vLe2yvt/1c7XZW69sF0KiJHMbvk7QyIo6TdIqky2wfL+lKSRsiYpGkDbXHALrUuGGPiP6IeLx2/w1JWyQdKek8SQfOpVwr6fxWNQmgee/rCzrbR0s6SdJGSXMjol8a+QdB0pw66yy33We7b0h7musWQMMmHHbbh0v6oaTLI2L3RNeLiNUR0RsRvdM0vZEeAVRgQmG3PU0jQb89Iu6tLR6wPa9WnydpZ2taBFCFcYfebFvSLZK2RMR1o0rrJF0saVXt9v6WdDgJTD36t4v1139vXrF+0d/+qFj/kw/dW6y30sr+8vDYz/+l/vBaz63/VVx31n6G1qo0kXH2pZK+Iukp25tqy67SSMjvtn2ppJckXdiaFgFUYdywR8TPJI05ubuks6ptB0CrcLoskARhB5Ig7EAShB1IgrADSXCJ6wRNnffRurXBNTOK6359wUPF+rKZAw31VIUVL59WrD9+U3nK5tk/2Fys97zBWHm3YM8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0mkGWff+wflny3e+6eDxfpVxzxQt3b2b73VUE9VGRh+u27t9HUri+se+1e/LNZ7XiuPk+8vVtFN2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJpxtm3nV/+d+3ZE+9p2bZvfG1hsX79Q2cX6x6u9+O+I4699sW6tUUDG4vrDhermEzYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEo6I8hPs+ZJuk/RRjVy+vDoirrd9jaQ/lvRK7alXRUT9i74lHeGeONlM/Aq0ysbYoN0xOOaJGRM5qWafpJUR8bjtmZIes72+VvteRHynqkYBtM5E5mfvl9Rfu/+G7S2Sjmx1YwCq9b4+s9s+WtJJkg6cg7nC9pO219ieVWed5bb7bPcNaU9TzQJo3ITDbvtwST+UdHlE7JZ0k6SFkhZrZM//3bHWi4jVEdEbEb3TNL2ClgE0YkJhtz1NI0G/PSLulaSIGIiI4YjYL+lmSUta1yaAZo0bdtuWdIukLRFx3ajl80Y97QJJ5ek8AXTURL6NXyrpK5Kesr2ptuwqSctsL5YUkrZJ+lpLOgRQiYl8G/8zSWON2xXH1AF0F86gA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJDHuT0lXujH7FUn/M2rRbEm72tbA+9OtvXVrXxK9NarK3o6KiI+MVWhr2N+zcbsvIno71kBBt/bWrX1J9NaodvXGYTyQBGEHkuh02Fd3ePsl3dpbt/Yl0Vuj2tJbRz+zA2ifTu/ZAbQJYQeS6EjYbZ9j+xnbz9u+shM91GN7m+2nbG+y3dfhXtbY3ml786hlPbbX236udjvmHHsd6u0a2y/X3rtNts/tUG/zbT9oe4vtp21/u7a8o+9doa+2vG9t/8xue4qkZyV9VtJ2SY9KWhYRv2hrI3XY3iapNyI6fgKG7dMlvSnptog4obbsHyUNRsSq2j+UsyLiii7p7RpJb3Z6Gu/abEXzRk8zLul8SV9VB9+7Ql9fVBvet07s2ZdIej4itkbEXkl3STqvA310vYh4WNLguxafJ2lt7f5ajfzP0nZ1eusKEdEfEY/X7r8h6cA04x197wp9tUUnwn6kpF+Nerxd3TXfe0j6ie3HbC/vdDNjmBsR/dLI/zyS5nS4n3cbdxrvdnrXNONd8941Mv15szoR9rGmkuqm8b+lEfEZSZ+TdFntcBUTM6FpvNtljGnGu0Kj0583qxNh3y5p/qjHH5e0owN9jCkidtRud0q6T903FfXAgRl0a7c7O9zP/+umabzHmmZcXfDedXL6806E/VFJi2wvsH2IpC9JWteBPt7D9ozaFyeyPUPS2eq+qajXSbq4dv9iSfd3sJd36JZpvOtNM64Ov3cdn/48Itr+J+lcjXwj/4Kkv+xED3X6+oSkJ2p/T3e6N0l3auSwbkgjR0SXSvqwpA2Snqvd9nRRb/8u6SlJT2okWPM61NtpGvlo+KSkTbW/czv93hX6asv7xumyQBKcQQckQdiBJAg7kARhB5Ig7EAShB1IgrADSfwfs4RxaLJFjqkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img_norm = X_train[0].reshape(28, 28)\n",
    "plt.imshow(img_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_train, y_train)]\n",
    "test_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_test, y_test)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [1.],\n",
       "       [0.],\n",
       "       [0.]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dt[0][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z:np.ndarray):\n",
    "    return 1.0 / (1.0 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define derivative of sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_der(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement the linear layer of our neural network\n",
    "<br>\n",
    "$$\n",
    "z = Wx + b\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(object):\n",
    "    \n",
    "    def __init__(self, din:int, dout:int, bias:bool=True):\n",
    "        super().__init__()\n",
    "        self.din = din\n",
    "        self.dout = dout\n",
    "        self.bias = bias\n",
    "        self.W = np.random.randn(dout, din)\n",
    "        self.b = np.random.randn(dout, 1)\n",
    "        self.a = None\n",
    "        self.z = None\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "        \n",
    "    def forward(self, a:np.ndarray):\n",
    "        self.a = a\n",
    "        self.z = self.W @ a + self.b\n",
    "        \n",
    "        return self.z\n",
    "    \n",
    "    def backward(self, delta_n:np.ndarray, z_p:np.ndarray):\n",
    "        sd = sigmoid_der(z_p)\n",
    "        delta = (self.W.T @ delta_n) * sd\n",
    "        self.der_W += delta_n @ self.a.T\n",
    "        self.der_b += delta_n\n",
    "        \n",
    "        return delta\n",
    "    \n",
    "    def zero_grad(self):\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return f'Linear({self.din}, {self.dout})'\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return self.__str__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here din is dimension of previous leyer and dout is the dimension of this layer\n",
    "<br>\n",
    "Store in der_W gradient of the cost function with this layer's weights and in der_b gradient with biases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also store previous layer's activation and current z in forward pass in order to calculate gradients with deltas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In \n",
    "```python\n",
    "def backward(self, delta_n:np.ndarray, z_p:np.ndarray):\n",
    "    sd = sigmoid_der(z_p)\n",
    "    delta = (self.W.T @ delta_n) * sd\n",
    "    self.der_W += delta_n @ self.a.T\n",
    "    self.der_b += delta_n\n",
    "\n",
    "    return delta\n",
    "``` \n",
    "we pass next layers $\\delta$ as delta_n and previous layers $z$ as z_p\n",
    "<br>\n",
    "Then calculate $\\sigma'(z^l_j)$ with \n",
    "```python \n",
    "sigmoid_der(z_p) \n",
    "```\n",
    "and $\\delta = ((w)^T \\delta) \\odot \\sigma'(z)$ for previous layer and we'll pass in to the previous layers backward function.\n",
    "<br>\n",
    "$$\\delta^l = ((w^{l+1})^T \\delta^{l+1}) \\odot \\sigma'(z^l)$$\n",
    "here \n",
    "- $w^{l+1}$ = self.W \n",
    "- $\\delta^{l+1}$ = delta_n\n",
    "- $z^l$ is z_p\n",
    "We also accumulate sum of the calculated gradients in der_W (for weights) and der_b (for biases) in order to use it in mean calculation in batch for gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def zero_grad(self):\n",
    "    self.der_W = np.zeros(self.W.shape)\n",
    "    self.der_b = np.zeros(self.b.shape)\n",
    "```\n",
    "Cleans accumulated gradients for next batch processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run feed forward our linear layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(884, 1)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.random.rand(28*28, 1)\n",
    "layer = Linear(28*28, 28*28 + 100)\n",
    "z = layer(x)\n",
    "a = sigmoid(z)\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can implement the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNetwork(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layers = list()\n",
    "        \n",
    "    def forward(self, *args, **kwargs):\n",
    "        pass\n",
    "    \n",
    "    def backward(self, x, y, out, **kwargs):\n",
    "        z_p = None,\n",
    "        delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "        for i in range(1, len(self.layers) + 1):\n",
    "            layer = self.layers[-i]\n",
    "            z_p = self.layers[-i - 1].z if i < len(self.layers) else x\n",
    "            delta = layer.backward(delta, z_p)\n",
    "            \n",
    "    def zero_grad(self):\n",
    "        for layer in self.layers:\n",
    "            layer.zero_grad()\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __setattr__(self, name, value):\n",
    "        super().__setattr__(name, value)\n",
    "        if not hasattr(self, 'layers'):\n",
    "            self.layers = []\n",
    "        if isinstance(value, Linear):\n",
    "            self.layers.append(value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here\n",
    "```python\n",
    "def __setattr__\n",
    "```\n",
    "will store all linear layers in list for further automatic processing\n",
    "the method:\n",
    "```python\n",
    "def backward(self, x, y, out, **kwargs):\n",
    "    z_p = None,\n",
    "    delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "    for i in range(1, len(self.layers) + 1):\n",
    "        layer = self.layers[-i]\n",
    "        z_p = self.layers[-i - 1].z if i < len(self.layers) else x\n",
    "        delta = layer.backward(delta, z_p)\n",
    "```\n",
    "First in will calculate delta for cost function:\n",
    "$$\n",
    "\\frac{1}{2n} \\sum_x \\| y(x) - a\\|^2.\n",
    "$$\n",
    "<br>\n",
    "$$\n",
    "\\delta^L = \\nabla_a C \\odot \\sigma'(z^L).\n",
    "$$\n",
    "by\n",
    "```python\n",
    "delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "```\n",
    "Then layer by layer from end to beginning runs backpropagation calls each layers backward function passing the next layers delta_n ($\\delta^{l+1}$) and previous layers z_p ($z^{l-1}$) and calculates deltas on the fly, layers accumulate gradients sum for weights and biases as we implemented above\n",
    "<br>\n",
    "The method\n",
    "```python\n",
    "def zero_grad\n",
    "```\n",
    "Cleans gradients by weighs and biases for each layer for next batch processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Simple feed forward model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNModel(NNetwork):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        #put layers here like\n",
    "        self.fc1 = Linear(784, 100)\n",
    "        self.fc2 = Linear(100, 100)\n",
    "        self.fc3 = Linear(100, 10)\n",
    "    \n",
    "        \n",
    "    def forward(self, x:np.ndarray):\n",
    "        # make the forward call\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = sigmoid(z1)\n",
    "        z2 = self.fc2(a1)\n",
    "        a2 = sigmoid(z2)\n",
    "        z3 = self.fc3(a2)\n",
    "        out = sigmoid(z3)\n",
    "        \n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Linear(784, 100), Linear(100, 100), Linear(100, 10)]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784, 1)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run our model on randomly generated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model(x)\n",
    "#pred\n",
    "out = np.argmax(pred)\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize random label and test backward (backpropagation) and zero_grad methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 1)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0]).reshape(10, 1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.backward(x, y, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15, 784) (15, 1)\n",
      "(10, 15) (10, 1)\n"
     ]
    }
   ],
   "source": [
    "for ll in model.layers:\n",
    "    print(ll.der_W.shape, ll.der_b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0\n",
      "0 0\n"
     ]
    }
   ],
   "source": [
    "model.zero_grad()\n",
    "for ll in model.layers:\n",
    "    print(np.count_nonzero(ll.der_W), np.count_nonzero(ll.der_b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to implement the loss function and gradient descent algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD(object):\n",
    "    \n",
    "    def __init__(self, lr):\n",
    "        self.lr = lr\n",
    "        \n",
    "    def optimize(self, model, batch):\n",
    "        for x, y in batch:\n",
    "            y_hat = model(x)\n",
    "            model.backward(x, y, y_hat)\n",
    "        for layer in model.layers:\n",
    "            layer.W = layer.W - self.lr * (layer.der_W / len(batch))\n",
    "        model.zero_grad()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The stochastic gradient descent will calculate mean of gradients in batch then makes step for gradient descent with learning rate and cleans accumulated gradient s for next batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = SGD(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After each epoch we'll run validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, test_data):\n",
    "    y_hats = [(np.argmax(model(x)), np.argmax(y)) for (x, y) in test_data]\n",
    "    valid_res = sum(int(x == y) for (x, y) in y_hats)\n",
    "    \n",
    "    return valid_res\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we call model (with updated weights and biases) and compare it's output with labels and count correct predictions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, training_data, test_data, epochs:int=12, batch_size:int=16):\n",
    "        n_test = len(test_data)\n",
    "        n = len(training_data)\n",
    "        for j in range(epochs):\n",
    "            random.shuffle(training_data)\n",
    "            batches = [training_data[k:k + batch_size] for k in range(0, n, batch_size)]\n",
    "            for batch in batches:\n",
    "                 optimizer.optimize(model, batch)       \n",
    "            if test_data:\n",
    "                valid_res = validate(model, test_data)\n",
    "                print(f'Epoch {j + 1}: {valid_res} / {n_test}, accuracy = {valid_res / n_test}')\n",
    "            else:\n",
    "                print(f'Epoch {j + 1} complete')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The training methods will iterate over the epochs, shuffle before each epoch data, group them in batches, run gradient descent per batch and validate model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/levan/anaconda3/envs/edu/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in exp\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: 4748 / 10000, accuracy = 0.4748\n",
      "Epoch 1: 5781 / 10000, accuracy = 0.5781\n",
      "Epoch 2: 6027 / 10000, accuracy = 0.6027\n",
      "Epoch 3: 6688 / 10000, accuracy = 0.6688\n",
      "Epoch 4: 7815 / 10000, accuracy = 0.7815\n",
      "Epoch 5: 7976 / 10000, accuracy = 0.7976\n",
      "Epoch 6: 8071 / 10000, accuracy = 0.8071\n",
      "Epoch 7: 8125 / 10000, accuracy = 0.8125\n",
      "Epoch 8: 8160 / 10000, accuracy = 0.816\n",
      "Epoch 9: 8174 / 10000, accuracy = 0.8174\n",
      "Epoch 10: 8202 / 10000, accuracy = 0.8202\n",
      "Epoch 11: 8237 / 10000, accuracy = 0.8237\n",
      "Epoch 12: 8230 / 10000, accuracy = 0.823\n",
      "Epoch 13: 8246 / 10000, accuracy = 0.8246\n",
      "Epoch 14: 8276 / 10000, accuracy = 0.8276\n",
      "Epoch 15: 8295 / 10000, accuracy = 0.8295\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, training_dt, test_dt, \n",
    "      epochs=16, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}