LSTM implementation without using nn.LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "task7.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdiZP6Ve_biv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision.datasets as dsets\n",
        "from torch.autograd import Variable\n",
        "from torch.nn import Parameter\n",
        "from torch import Tensor\n",
        "import math\n",
        "\n",
        "\n",
        "cuda = True if torch.cuda.is_available() else False\n",
        "    \n",
        "Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor    \n",
        "\n",
        "torch.manual_seed(1)\n",
        "if torch.cuda.is_available():\n",
        "    torch.cuda.manual_seed_all(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B5eRrNpq_mJU",
        "colab_type": "code",
        "outputId": "aa6e52d8-5ab0-4c8e-a0e7-8534985b7d18",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "#Loading Dataset\n",
        "train_dataset = dsets.MNIST(root='./data', \n",
        "                            train=True, \n",
        "                            transform=transforms.ToTensor(),\n",
        "                            download=True)\n",
        " \n",
        "test_dataset = dsets.MNIST(root='./data', \n",
        "                           train=False, \n",
        "                           transform=transforms.ToTensor())\n",
        " \n",
        "batch_size = 100\n",
        "n_iters = 6000\n",
        "num_epochs = n_iters / (len(train_dataset) / batch_size)\n",
        "num_epochs = int(num_epochs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/9912422 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "9920512it [00:00, 21198779.70it/s]                            \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "32768it [00:00, 329525.81it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n",
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1654784it [00:00, 5356523.53it/s]                           \n",
            "8192it [00:00, 131874.89it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n",
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9WYQU_j_qeb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Loading the data into train_loader and test_loader\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
        "                                           batch_size=batch_size, \n",
        "                                           shuffle=True)\n",
        " \n",
        "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
        "                                          batch_size=batch_size, \n",
        "                                          shuffle=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1oEsbUnthoJp",
        "colab_type": "text"
      },
      "source": [
        "#LSTM Implementation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CY2Kttx9DDp6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTM(nn.Module):\n",
        "\n",
        "    def __init__(self, in_sz, hid_sz, bias=True):\n",
        "        super(LSTM, self).__init__()\n",
        "        self.in_sz = in_sz\n",
        "        self.hid_sz = hid_sz\n",
        "        self.bias = bias\n",
        "        self.layer1 = nn.Linear(in_sz, 4 * hid_sz, bias=bias)\n",
        "        self.layer2 = nn.Linear(hid_sz, 4 * hid_sz, bias=bias)\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        std = 1.0 / math.sqrt(self.hid_sz)\n",
        "        for w in self.parameters():\n",
        "            w.data.uniform_(-std, std)\n",
        "    \n",
        "    def forward(self, x, hidden):\n",
        "        \n",
        "        hx, cx = hidden\n",
        "        x = x.view(-1, x.size(1))\n",
        "        gates = self.layer1(x) + self.layer2(hx)\n",
        "        gates = gates.squeeze()\n",
        "        ingate, forgetgate, nextgate, outgate = gates.chunk(4, 1)\n",
        "        ingate = torch.sigmoid(ingate)\n",
        "        forgetgate = torch.sigmoid(forgetgate)\n",
        "        nextgate = torch.tanh(nextgate)\n",
        "        outgate = torch.sigmoid(outgate)\n",
        "        cy = torch.mul(cx, forgetgate) +  torch.mul(ingate, nextgate)        \n",
        "        hy = torch.mul(outgate, torch.tanh(cy))\n",
        "        return (hy, cy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_r-Y--aDLVC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Create LSTM model class\n",
        " \n",
        "class LSTMModel(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, bias=True):\n",
        "        super(LSTMModel, self).__init__()\n",
        "        # Hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "        # Number of hidden layers\n",
        "        self.layer_dim = layer_dim       \n",
        "        self.lstm = LSTM(input_dim, hidden_dim, layer_dim)  \n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "     \n",
        "    \n",
        "    \n",
        "    def forward(self, x):\n",
        "        # Initialize hidden state with zeros\n",
        "        if torch.cuda.is_available():\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
        "        else:\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
        "        # Initialize cell state\n",
        "        if torch.cuda.is_available():\n",
        "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
        "        else:\n",
        "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), hidden_dim))\n",
        "        outs = []\n",
        "        cn = c0[0,:,:]\n",
        "        hn = h0[0,:,:]\n",
        "        # 28 time steps\n",
        "        for seq in range(x.size(1)):\n",
        "            hn, cn = self.lstm(x[:,seq,:], (hn,cn)) \n",
        "            outs.append(hn)\n",
        "        # Index hidden state of last time step    \n",
        "        out = outs[-1].squeeze()\n",
        "        out = self.fc(out) \n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "em9y5lzkDU1b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Instantiate model class for LSTM\n",
        "input_dim = 28\n",
        "hidden_dim = 128\n",
        "layer_dim = 1  \n",
        "output_dim = 10\n",
        "model_LSTM = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    model_LSTM.cuda()    \n",
        "criterion = nn.CrossEntropyLoss()\n",
        "learning_rate = 0.1\n",
        "optimizer = torch.optim.SGD(model_LSTM.parameters(), lr=learning_rate)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rbn9PJ2FDYXr",
        "colab_type": "code",
        "outputId": "f95cc784-7c19-4d7e-95fc-4d588b886032",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "#Train the model\n",
        "\n",
        "# Number of steps to unroll\n",
        "seq_dim = 28 \n",
        "\n",
        "loss_list = []\n",
        "iter = 0\n",
        "for epoch in range(num_epochs):\n",
        "    for i, (images, labels) in enumerate(train_loader):\n",
        "        # Load images as Variable\n",
        "                  \n",
        "        if torch.cuda.is_available():\n",
        "            images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "            labels = Variable(labels.cuda())\n",
        "        else:\n",
        "            images = Variable(images.view(-1, seq_dim, input_dim))\n",
        "            labels = Variable(labels)\n",
        "          \n",
        "        # Clear gradients w.r.t. parameters\n",
        "        optimizer.zero_grad()\n",
        "         \n",
        "        # Forward pass to get output/logits\n",
        "        # outputs.size() --> 100, 10\n",
        "        outputs = model_LSTM(images)\n",
        "\n",
        "        # Calculate Loss: softmax --> cross entropy loss\n",
        "        loss = criterion(outputs, labels)\n",
        "\n",
        "        if torch.cuda.is_available():\n",
        "            loss.cuda()\n",
        "\n",
        "        # Getting gradients w.r.t. parameters\n",
        "        loss.backward()\n",
        "\n",
        "        # Updating parameters\n",
        "        optimizer.step()\n",
        "        loss_list.append(loss.item())\n",
        "        iter += 1\n",
        "         \n",
        "        if iter % 500 == 0:\n",
        "            # Calculate Accuracy         \n",
        "            correct = 0\n",
        "            total = 0\n",
        "            # Iterate through test dataset\n",
        "            for images, labels in test_loader:\n",
        "                \n",
        "                if torch.cuda.is_available():\n",
        "                    images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "                else:\n",
        "                    images = Variable(images.view(-1 , seq_dim, input_dim))\n",
        "                \n",
        "                # Forward pass only to get logits/output\n",
        "                outputs = model_LSTM(images)\n",
        "                \n",
        "                # Get predictions from the maximum value\n",
        "                _, predicted = torch.max(outputs.data, 1)\n",
        "                 \n",
        "                # Total number of labels\n",
        "                total += labels.size(0)\n",
        "                 \n",
        "                # Total correct predictions\n",
        "                \n",
        "                if torch.cuda.is_available():\n",
        "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
        "                else:\n",
        "                    correct += (predicted == labels).sum()\n",
        "             \n",
        "            accuracy = 100 * correct / total\n",
        "             \n",
        "            # Print Loss\n",
        "            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))\n",
        "            "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration: 500. Loss: 2.272420644760132. Accuracy: 18\n",
            "Iteration: 1000. Loss: 1.1450212001800537. Accuracy: 61\n",
            "Iteration: 1500. Loss: 0.6364210247993469. Accuracy: 83\n",
            "Iteration: 2000. Loss: 0.39562085270881653. Accuracy: 91\n",
            "Iteration: 2500. Loss: 0.14795413613319397. Accuracy: 94\n",
            "Iteration: 3000. Loss: 0.0666346624493599. Accuracy: 95\n",
            "Iteration: 3500. Loss: 0.16048261523246765. Accuracy: 95\n",
            "Iteration: 4000. Loss: 0.1573454737663269. Accuracy: 96\n",
            "Iteration: 4500. Loss: 0.11004464328289032. Accuracy: 96\n",
            "Iteration: 5000. Loss: 0.13709653913974762. Accuracy: 96\n",
            "Iteration: 5500. Loss: 0.013900227844715118. Accuracy: 97\n",
            "Iteration: 6000. Loss: 0.04884167015552521. Accuracy: 97\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yB76TN1-2bGe",
        "colab_type": "text"
      },
      "source": [
        "#Implementation of GRU \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fnpXxlw8Di0O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Implementation of GRU Cell\n",
        "\n",
        "class GRU(nn.Module):\n",
        "\n",
        "  def __init__(self, in_sz, hid_sz, bias=True):\n",
        "        super(GRU, self).__init__()\n",
        "        self.in_sz = in_sz\n",
        "        self.hid_sz = hid_sz\n",
        "        self.bias = bias\n",
        "        self.layer1 = nn.Linear(in_sz, 3 * hid_sz, bias=bias)\n",
        "        self.layer2 = nn.Linear(hid_sz, 3 * hid_sz, bias=bias)\n",
        "        self.reset_parameters()\n",
        "\n",
        "\n",
        "\n",
        "  def reset_parameters(self):\n",
        "        std = 1.0 / math.sqrt(self.hid_sz)\n",
        "        for w in self.parameters():\n",
        "            w.data.uniform_(-std, std)\n",
        "    \n",
        "  def forward(self, x, hidden):\n",
        "        \n",
        "        x = x.view(-1, x.size(1))\n",
        "        \n",
        "        gate_x = self.layer1(x) \n",
        "        gate_h = self.layer2(hidden)\n",
        "        \n",
        "        gate_x = gate_x.squeeze()\n",
        "        gate_h = gate_h.squeeze()\n",
        "        i_r, i_i, i_n = gate_x.chunk(3, 1)\n",
        "        h_r, h_i, h_n = gate_h.chunk(3, 1)\n",
        "        \n",
        "        \n",
        "        reset_gate = torch.sigmoid(i_r + h_r)\n",
        "        input_gate = torch.sigmoid(i_i + h_i)\n",
        "        next_gate = torch.tanh(i_n + (reset_gate * h_n))\n",
        "        \n",
        "        hy = next_gate + input_gate * (hidden - next_gate)\n",
        "        \n",
        "        \n",
        "        return hy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uZaTWPrdDmjR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#create GRU model class\n",
        "class GRUModel(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, bias=True):\n",
        "        super(GRUModel, self).__init__()\n",
        "        # Hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.layer_dim = layer_dim\n",
        "         \n",
        "       \n",
        "        self.gru = GRU(input_dim, hidden_dim, layer_dim)\n",
        "        \n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "     \n",
        "    \n",
        "    \n",
        "    def forward(self, x):\n",
        "        # Initialize hidden state with zeros\n",
        "        if torch.cuda.is_available():\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
        "        else:\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
        "         \n",
        "       \n",
        "        outs = []\n",
        "        \n",
        "        hn = h0[0,:,:]\n",
        "        #for 28 time steps\n",
        "        for seq in range(x.size(1)):\n",
        "            hn = self.gru(x[:,seq,:], hn) \n",
        "            outs.append(hn)\n",
        "            \n",
        "        # Index hidden state of last time step\n",
        "        out = outs[-1].squeeze()\n",
        "        \n",
        "        out = self.fc(out) \n",
        "        # out.size() --> 100, 10\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nGAUZo3CDt4F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Instantiate model class for GRU Model\n",
        "input_dim = 28\n",
        "hidden_dim = 128\n",
        "layer_dim = 1  \n",
        "output_dim = 10\n",
        " \n",
        "model_GRU = GRUModel(input_dim, hidden_dim, layer_dim, output_dim)\n",
        "\n",
        " \n",
        "if torch.cuda.is_available():\n",
        "    model_GRU.cuda()\n",
        "     \n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        " \n",
        "learning_rate = 0.1  #learning rate\n",
        " \n",
        "optimizer = torch.optim.SGD(model_GRU.parameters(), lr=learning_rate)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nSTjFcKSDxUd",
        "colab_type": "code",
        "outputId": "1a13d978-139c-466a-e1d3-25c7b64c4667",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "#Traing the model\n",
        "# Number of steps to unroll\n",
        "seq_dim = 28 \n",
        "\n",
        "loss_list = []\n",
        "iter = 0\n",
        "for epoch in range(num_epochs):\n",
        "    for i, (images, labels) in enumerate(train_loader):\n",
        "          \n",
        "        if torch.cuda.is_available():\n",
        "            images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "            labels = Variable(labels.cuda())\n",
        "        else:\n",
        "            images = Variable(images.view(-1, seq_dim, input_dim))\n",
        "            labels = Variable(labels)\n",
        "          \n",
        "        # Clear gradients w.r.t. parameters\n",
        "        optimizer.zero_grad()\n",
        "         \n",
        "        # Forward pass to get output/logits\n",
        "        # outputs.size() --> 100, 10\n",
        "        outputs = model_GRU(images)\n",
        "\n",
        "        # Calculate Loss: softmax --> cross entropy loss\n",
        "        loss = criterion(outputs, labels)\n",
        "\n",
        "        if torch.cuda.is_available():\n",
        "            loss.cuda()\n",
        "\n",
        "        # Getting gradients w.r.t. parameters\n",
        "        loss.backward()\n",
        "\n",
        "        # Updating parameters\n",
        "        optimizer.step()\n",
        "        \n",
        "        loss_list.append(loss.item())\n",
        "        iter += 1\n",
        "         \n",
        "        if iter % 500 == 0:\n",
        "            # Calculate Accuracy         \n",
        "            correct = 0\n",
        "            total = 0\n",
        "            # Iterate through test dataset\n",
        "            for images, labels in test_loader:\n",
        "\n",
        "                if torch.cuda.is_available():\n",
        "                    images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "                else:\n",
        "                    images = Variable(images.view(-1 , seq_dim, input_dim))\n",
        "                \n",
        "                # Forward pass only to get logits/output\n",
        "                outputs = model_GRU(images)\n",
        "                \n",
        "                # Get predictions from the maximum value\n",
        "                _, predicted = torch.max(outputs.data, 1)\n",
        "                 \n",
        "                # Total number of labels\n",
        "                total += labels.size(0)\n",
        "\n",
        "                if torch.cuda.is_available():\n",
        "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
        "                else:\n",
        "                    correct += (predicted == labels).sum()\n",
        "             \n",
        "            accuracy = 100 * correct / total\n",
        "             \n",
        "            # Print Loss\n",
        "            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration: 500. Loss: 1.2861151695251465. Accuracy: 54\n",
            "Iteration: 1000. Loss: 0.5903604626655579. Accuracy: 78\n",
            "Iteration: 1500. Loss: 0.3193312883377075. Accuracy: 91\n",
            "Iteration: 2000. Loss: 0.26455608010292053. Accuracy: 94\n",
            "Iteration: 2500. Loss: 0.17853772640228271. Accuracy: 95\n",
            "Iteration: 3000. Loss: 0.14693112671375275. Accuracy: 95\n",
            "Iteration: 3500. Loss: 0.09304679930210114. Accuracy: 96\n",
            "Iteration: 4000. Loss: 0.12386438995599747. Accuracy: 96\n",
            "Iteration: 4500. Loss: 0.027463655918836594. Accuracy: 97\n",
            "Iteration: 5000. Loss: 0.11559031158685684. Accuracy: 97\n",
            "Iteration: 5500. Loss: 0.06111504137516022. Accuracy: 97\n",
            "Iteration: 6000. Loss: 0.04078087955713272. Accuracy: 97\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h90dXK8KDE-a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}