workshop_4_optimizations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create anaconda environment\n",
    "<br>\n",
    "```bash\n",
    "conda create -n ml python=3.7.5 jupyter\n",
    "```\n",
    "Install fastai library\n",
    "<br>\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y_str = fetch_openml('mnist_784', version=1, return_X_y=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img1 = X[1].reshape(28, 28)\n",
    "plt.imshow(img1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_int = y_str.astype(int)\n",
    "\n",
    "def encode_idx(idx):\n",
    "    cd = np.zeros(10)\n",
    "    cd[idx] = 1\n",
    "    \n",
    "    return cd\n",
    "    \n",
    "y_v = np.array([encode_idx(i) for i in y_int])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_norm = X / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = X_norm[:60000], X_norm[60000:]\n",
    "y_train, y_test = y_v[:60000], y_v[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "img_norm = X_train[0].reshape(28, 28)\n",
    "plt.imshow(img_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_train, y_train)]\n",
    "test_dt = [(x.reshape(784, 1), y.reshape(10, 1)) for (x, y) in zip(X_test, y_test)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import gzip\n",
    "import pickle as pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data')\n",
    "mnist = path / 'mnist'\n",
    "mnist.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt_path = mnist / 'training_dt.pickle'\n",
    "test_dt_path = mnist / 'test_dt.pickle'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_data(data_path, data_set):\n",
    "    with data_path.open(mode='wb') as fl:\n",
    "        pkl.dump(data_set, fl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(data_path):\n",
    "    with data_path.open(mode='rb') as fl:\n",
    "        data_set = pkl.load(fl)\n",
    "    \n",
    "    return data_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_data(training_dt_path, training_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_data(test_dt_path, test_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt_ld = load_data(training_dt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dt_ld = load_data(test_dt_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z:np.ndarray):\n",
    "    return 1.0 / (1.0 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define derivative of sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_der(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement the linear layer of our neural network\n",
    "<br>\n",
    "$$\n",
    "z = Wx + b\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(object):\n",
    "    \n",
    "    def __init__(self, din:int, dout:int, bias:bool=True):\n",
    "        super().__init__()\n",
    "        self.din = din\n",
    "        self.dout = dout\n",
    "        self.bias = bias\n",
    "        self.W = np.random.randn(dout, din)\n",
    "        self.b = np.random.randn(dout, 1)\n",
    "        self.a = None\n",
    "        self.z = None\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "        \n",
    "    def forward(self, a:np.ndarray):\n",
    "        self.a = a\n",
    "        self.z = self.W @ a + self.b\n",
    "        \n",
    "        return self.z\n",
    "    \n",
    "    def backward(self, delta_n:np.ndarray, z_p:np.ndarray):\n",
    "        sd = sigmoid_der(z_p)\n",
    "        delta = (self.W.T @ delta_n) * sd\n",
    "        self.der_W += delta_n @ self.a.T\n",
    "        self.der_b += delta_n\n",
    "        \n",
    "        return delta\n",
    "    \n",
    "    def zero_grad(self):\n",
    "        self.der_W = np.zeros(self.W.shape)\n",
    "        self.der_b = np.zeros(self.b.shape)\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return f'Linear({self.din}, {self.dout})'\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return self.__str__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can implement the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNetwork(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layers = list()\n",
    "        self.acts = list()\n",
    "        self.layer_dict = OrderedDict()\n",
    "        \n",
    "    def forward(self, *args, **kwargs):\n",
    "        pass\n",
    "    \n",
    "    def backward(self, x, y, out, **kwargs):\n",
    "        z_p = None,\n",
    "        delta = (out - y) * sigmoid_der(self.layers[-1].z)\n",
    "        for i in range(1, len(self.layers) + 1):\n",
    "            layer = self.layers[-i]\n",
    "            z_p = self.layers[-i - 1].z if i < len(self.layers) else x\n",
    "            delta = layer.backward(delta, z_p)\n",
    "            \n",
    "    def zero_grad(self):\n",
    "        for layer in self.layers:\n",
    "            layer.zero_grad()\n",
    "            \n",
    "    def save(self, filename:str):\n",
    "        weight_dict = dict()\n",
    "        for k, v in self.layer_dict.items():\n",
    "            weight_dict[k] = [v.W, v.b]\n",
    "        with open(filename, 'wb') as handle:\n",
    "            pickle.dump(weight_dict, handle)\n",
    "            \n",
    "    def load(self, filename:str):\n",
    "        with open(filename, 'rb') as handle:\n",
    "            weight_dict = pickle.load(handle)\n",
    "        for k, (W, b) in weight_dict.items():\n",
    "            if k in self.layer_dict:\n",
    "                self.layer_dict[k].W = W\n",
    "                self.layer_dict[k].b = b\n",
    "    \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self.forward(*args, **kwargs)\n",
    "    \n",
    "    def __setattr__(self, name, value):\n",
    "        super().__setattr__(name, value)\n",
    "        if not hasattr(self, 'layers'):\n",
    "            self.layers = []\n",
    "        if not hasattr(self, 'layer_dict'):\n",
    "            self.layer_dict = OrderedDict()\n",
    "        if isinstance(value, Linear):\n",
    "            self.layers.append(value)\n",
    "            self.layer_dict[name] = value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Simple feed forward model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNModel(NNetwork):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = Linear(28*28, 100)\n",
    "        self.fc2 = Linear(100, 50)\n",
    "        self.fc3 = Linear(50, 10)\n",
    "        \n",
    "    def forward(self, x:np.ndarray):\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = sigmoid(z1)\n",
    "        z2 = self.fc2(a1)\n",
    "        a2 = sigmoid(z2)\n",
    "        z3 = self.fc3(a2)\n",
    "        out = sigmoid(z3)\n",
    "        \n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layer_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize random label and test backward (backpropagation) and zero_grad methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.random.rand(28*28, 1)\n",
    "pred = model(x)\n",
    "out = np.argmax(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0]).reshape(10, 1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.backward(x, y, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ll in model.layers:\n",
    "    print(ll.der_W.shape, ll.der_b.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGD with momentum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Momentum implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "beta=0.9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to implement the loss function and gradient descent algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD(object):\n",
    "    \n",
    "    def __init__(self, lr, momentum=None):\n",
    "        self.lr = lr\n",
    "        self.beta= momentum\n",
    "        self.v_w = dict()\n",
    "        self.v_b = dict()\n",
    "        \n",
    "    def optimize(self, model, batch):\n",
    "        for x, y in batch:\n",
    "            y_hat = model(x)\n",
    "            model.backward(x, y, y_hat)\n",
    "        glob_w = 0.0\n",
    "        glob_b = 0.0\n",
    "        for layer in model.layers:\n",
    "            max_w = np.max(np.absolute(layer.der_W))\n",
    "            max_b = np.max(np.absolute(layer.der_b))\n",
    "            glob_w = max(max_w, glob_w)\n",
    "            glob_b = max(max_b, glob_b)\n",
    "            if self.beta:\n",
    "                self.v_w.setdefault(layer, np.zeros(layer.der_W.shape))\n",
    "                self.v_b.setdefault(layer, np.zeros(layer.der_b.shape))\n",
    "                self.v_w[layer] = self.beta * self.v_w[layer] + (1 - beta) * layer.der_W\n",
    "                self.v_b[layer] = self.beta * self.v_b[layer] + (1 - beta) * layer.der_b\n",
    "                layer.W = layer.W - self.lr * (self.v_w[layer])\n",
    "                layer.b = layer.b - self.lr * (self.v_b[layer])\n",
    "            else:\n",
    "                layer.W = layer.W - self.lr * (layer.der_W / len(batch))\n",
    "                layer.b = layer.b - self.lr * (layer.der_b / len(batch))\n",
    "        model.zero_grad()\n",
    "        \n",
    "        return glob_w, glob_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The stochastic gradient descent will calculate mean of gradients in batch then makes step for gradient descent with learning rate and cleans accumulated gradient s for next batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = SGD(0.4, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNModel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After each epoch we'll run validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(test_data):\n",
    "    y_hats = [(np.argmax(model(x)), np.argmax(y)) for (x, y) in test_data]\n",
    "    valid_res = sum(int(pred == y) for (pred, y) in y_hats)\n",
    "    \n",
    "    return valid_res\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we call model (with updated weights and biases) and compare it's output with labels and count correct predictions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, training_data, test_data, epochs:int=12, batch_size:int=16, verbose:bool=False):\n",
    "        if test_data: n_test = len(test_data)\n",
    "        n = len(training_data)\n",
    "        glob_w, glob_b = 0.0, 0.0\n",
    "        for j in range(epochs):\n",
    "            random.shuffle(training_data)\n",
    "            batches = [training_data[k:k + batch_size] for k in range(0, n, batch_size)]\n",
    "            for batch in batches:\n",
    "                max_w, max_b = optimizer.optimize(model, batch)\n",
    "                glob_w, glob_b = max(max_w, glob_w), max(max_b, glob_b)\n",
    "            if verbose:\n",
    "                print(f'max_w = {glob_w}, max_b = {glob_b}')\n",
    "            if test_data:\n",
    "                valid_res = validate(test_data)\n",
    "                print(f'Epoch {j}: {valid_res} / {n_test}, accuracy = {valid_res / n_test}')\n",
    "            else:\n",
    "                print(f'Epoch {j} complete')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The training methods will iterate over the epochs, shuffle before each epoch data, group them in batches, run gradient descent per batch and validate model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train(model, optimizer, training_dt, test_dt, epochs=12, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serialize the Trained Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data') / 'weights.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(str(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dsk = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_1 = model_dsk.layers[0].W.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dsk.load(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_2 = model_dsk.layers[0].W.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.all(W_1 == W_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch and FastAI libraries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An open source machine learning framework that accelerates the path from research prototyping to production deployment. \n",
    "[PyTorch](https://pytorch.org/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install only PyTorch library:\n",
    "\n",
    "```bash\n",
    "conda install pytorch torchvision -c pytorch\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install only PyTorch library:\n",
    "\n",
    "```bash\n",
    "pip install torch torchvision\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The fastai deep learning library is more like scikit-learn on top of PyTorch\n",
    "[FastAI](https://www.fast.ai/)\n",
    "[FastAI GitHub](https://github.com/fastai/fastai)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install with fastai library:\n",
    "```bash\n",
    "conda install -c pytorch -c fastai fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install with fastai library:\n",
    "```bash\n",
    "pip install fastai\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchvision import datasets, transforms\n",
    "from torch import nn, optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "t = torch.Tensor(64, 28, 28)\n",
    "print(t.size())\n",
    "t_v = t.view(t.size(0), -1)\n",
    "t_v.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "class FlattenTensor(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def flatten(self, x):\n",
    "        return x.view(-1, 784)\n",
    "        \n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return flatten(*args, **kwargs)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "?? transforms.ToTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([transforms.ToTensor(),\n",
    "                                transforms.Normalize((0.5,), (0.5,))])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "For the preprocessing our data, we will use composition of the builtin functions:\n",
    "\n",
    "```python\n",
    "transforms.ToTensor()\n",
    "```\n",
    "converts NumPy array or PIL image in PyTorch tensor\n",
    "\n",
    "```python\n",
    "transforms.Normalize()\n",
    "```\n",
    "Normalize our data with mean 0.5 and variance 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data')\n",
    "mnist_tensors = path / 'mnist_tensors'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trainset = datasets.MNIST(str(mnist_tensors), download=True, train=True, transform=transform)\n",
    "valset = datasets.MNIST(str(mnist_tensors), download=True, train=False, transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True, num_workers=0)\n",
    "valloader = torch.utils.data.DataLoader(valset, batch_size=64, shuffle=True, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We initialized dataset with transforms and generated dataloader which will shuffle data and load it per batches:\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameters for datasets.MNIST:\n",
    "- path to download\n",
    "- download: if data should be downloaded\n",
    "- train: is it training set or validation set\n",
    "- transform: transformations (preprocessing)\n",
    "\n",
    "<br>\n",
    "Parameters for torch.utils.data.DataLoader:\n",
    "- dataset which should be loaded iteratively\n",
    "- batch_size: mini-batch size\n",
    "- shuffle: shuffle data per batch\n",
    "- num_workers: number of threads for data loading\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameters for datasets.MNIST:\n",
    "- path to download\n",
    "- download: if data should be downloaded\n",
    "- train: is it training set or validation set\n",
    "- transform: transformations (preprocessing)\n",
    "\n",
    "<br>\n",
    "Parameters for torch.utils.data.DataLoader:\n",
    "- dataset which should be loaded iteratively\n",
    "- batch_size: mini-batch size\n",
    "- shuffle: shuffle data per batch\n",
    "- num_workers: number of threads for data loading\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Look into the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_iter = iter(trainloader)\n",
    "images, labels = data_iter.next()\n",
    "\n",
    "print(images.shape)\n",
    "print(labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.imshow(images[6].numpy().squeeze(), cmap='gray_r');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plt.figure()\n",
    "num_of_images = 60\n",
    "for index in range(1, num_of_images + 1):\n",
    "    plt.subplot(6, 10, index)\n",
    "    plt.axis('off')\n",
    "    plt.imshow(images[index].numpy().squeeze(), cmap='gray_r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNModel(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(28*28, 100)\n",
    "        self.ac1 = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(100, 50)\n",
    "        self.ac2 = nn.ReLU()\n",
    "        self.fc3 = nn.Linear(50, 10)\n",
    "        self.ac3 = nn.LogSoftmax(dim=1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = self.ac1(z1)\n",
    "        z2 = self.fc2(a1)\n",
    "        a2 = self.ac2(z2)\n",
    "        z3 = self.fc3(a2)\n",
    "        out = self.ac3(z3)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ?? nn.Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ?? nn.Linear"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LogSoftmax:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instead of using the sigmoid let's define more probabilistic function:\n",
    "$$\n",
    "softmax(x) = \\frac{e^{x_k}}{\\sum_{i}e^{x_i}}\n",
    "$$\n",
    "and \n",
    "$$\n",
    "logsoftmax(x) = log(\\frac{e^{x_k}}{\\sum_{i}e^{x_i}})\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "softmax = F.softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ?? nn.NLLLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#list(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare training iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, valloader):\n",
    "    model.eval()\n",
    "    correct, total = 0.0, 0.0\n",
    "    for images, labels in valloader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        imgs = images.view(-1, 784)\n",
    "        with torch.no_grad():\n",
    "            logits = model(imgs)\n",
    "        _, preds = torch.max(logits.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (preds == labels).sum().item()\n",
    "    print(\"\\nValidation Accuracy =\", (correct / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, trainloader, valloader, criterion, optimizer, valid_funct=validate, epochs=16, val_step=10):\n",
    "    for e in range(epochs):\n",
    "        running_loss = 0\n",
    "        start_time = time()\n",
    "        for images, labels in trainloader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            model.train()\n",
    "            # Flatten MNIST images into a 784 long vector\n",
    "            images = images.view(images.shape[0], -1)\n",
    "\n",
    "            # Training pass\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            output = model(images)\n",
    "            loss = criterion(output, labels)\n",
    "\n",
    "            #This is where the model learns by backpropagating\n",
    "            loss.backward()\n",
    "\n",
    "            #And optimizes its weights here\n",
    "            optimizer.step()\n",
    "\n",
    "            running_loss += loss.item()\n",
    "        else:\n",
    "            print(f'Epoch {e + 1} - Training loss: {running_loss/len(trainloader)}')\n",
    "            model.eval()\n",
    "            valid_funct(model, valloader)\n",
    "    print(f'\\nTraining Time (in minutes) = {(time() - start_time) / 60}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train(model, trainloader, valloader, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train other model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNEmbedding(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(28*28, 100)\n",
    "        self.ac1 = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(100, 50)\n",
    "        self.ac2 = nn.ReLU()\n",
    "        self.fc3 = nn.Linear(50, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        z1 = self.fc1(x)\n",
    "        a1 = self.ac1(z1)\n",
    "        z2 = self.fc2(a1)\n",
    "        a2 = self.ac2(z2)\n",
    "        out = self.fc3(a2)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedd = NNEmbedding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "softmax = nn.Softmax(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_softmax(model, valloader):\n",
    "    model.eval()\n",
    "    correct_count, all_count = 0.0, 0.0\n",
    "    total = 0.0\n",
    "    correct = 0.0\n",
    "    for images, labels in valloader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        imgs = images.view(-1, 784)\n",
    "        with torch.no_grad():\n",
    "            logits = model(imgs)\n",
    "        _, preds = torch.max(logits.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (preds == labels).sum().item()\n",
    "    print(\"\\nModel Accuracy =\", (correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entr = nn.CrossEntropyLoss() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rms_prop = optim.RMSprop(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train(embedd, trainloader, valloader, cross_entr, rms_prop, valid_funct=validate_softmax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model with FastAI library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.basics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import gzip\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_fastai = path / 'mnist_fastai'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "! wget -P {mnist_fastai} http://deeplearning.net/data/mnist/mnist.pkl.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(mnist_fastai /'mnist.pkl.gz', 'rb') as f:\n",
    "    ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding='latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(x_train[6].reshape((28,28)), cmap='gray')\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x_train, y_train, x_valid, y_valid = map(torch.tensor, \n",
    "#                                      (x_train, y_train, x_valid, y_valid))\n",
    "x_train, y_train, x_valid, y_valid = [torch.tensor(d) for d in (x_train, y_train, x_valid, y_valid)]\n",
    "n, c = x_train.shape\n",
    "x_train.shape, y_train.min(), y_train.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "train_ds = TensorDataset(x_train, y_train)\n",
    "valid_ds = TensorDataset(x_valid, y_valid)\n",
    "data = DataBunch.create(train_ds, valid_ds, bs=bs, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(data.train_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(iter(data.train_dl))\n",
    "x.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(nn.Linear(28 * 28, 128),\n",
    "                    nn.ReLU(),\n",
    "                    nn.Linear(128, 64, bias=True),\n",
    "                    nn.ReLU(),\n",
    "                    nn.Linear(64, 10, bias=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(OrderedDict([\n",
    "                        ('fc1', nn.Linear(28 * 28, 128)),\n",
    "                        ('ac1', nn.ReLU()),\n",
    "                        ('fc2', nn.Linear(128, 64, bias=True)),\n",
    "                        ('ac2', nn.ReLU()),\n",
    "                        ('fc3', nn.Linear(64, 10, bias=True))]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#?? nn.Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ?? nn.Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, net, loss_func=loss_func, metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ?? Learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit_one_cycle(4, 2e-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make some analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}