modeling2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('DSL-StrongPasswordData.csv')\n",
    "time_stamp = data.iloc[:, 3:]\n",
    "typist = np.zeros(data.shape[0])\n",
    "typist_list = list(set(data.subject))\n",
    "for i, s in data.subject.iteritems():\n",
    "    typist[i] = typist_list.index(s)\n",
    "\n",
    "np_time_stamp = np.array(time_stamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "num_classes = 10\n",
    "# For the weights, we set requires_grad after the initialization,\n",
    "# since we don’t want that step included in the gradient. \n",
    "# Note that a trailling _ in PyTorch signifies that the operation is performed in-place.\n",
    "weights = torch.randn(784, 10)\n",
    "weights.requires_grad_()\n",
    "bias = torch.zeros(10, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    return x.exp() / x.exp().sum(-1).unsqueeze(-1)\n",
    "\n",
    "def model(xb):\n",
    "    # @ means matrix multiplication\n",
    "    return softmax(xb @ weights + bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([nan, 0., 0., nan, 0., 0., 0., 0., nan, nan])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake = np.random.random(784) * 10\n",
    "fake_tensor = torch.tensor(fake, dtype=torch.int8).float()\n",
    "with torch.no_grad():\n",
    "    # output = model(fake)\n",
    "    # TypeError: unsupported operand type(s) for @: 'numpy.ndarray' and 'Tensor'\n",
    "    output = model(fake_tensor)\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 91.3079, -87.6783,  -9.5313, 145.9026, -39.3080,  69.0123, -13.5331,\n",
       "          87.0673, 145.2812, 135.5267], grad_fn=<AddBackward0>),\n",
       " tensor([[ 0.7174,  0.9600,  0.8354,  ...,  0.6415,  0.1312,  0.2727],\n",
       "         [ 1.5002, -0.7876, -0.8425,  ...,  0.8735,  0.5796, -0.3114],\n",
       "         [-1.2711, -1.5224,  0.9312,  ...,  1.1718,  0.7821, -0.0384],\n",
       "         ...,\n",
       "         [ 0.9191,  0.3357,  0.3700,  ...,  1.7623,  0.8601,  0.2099],\n",
       "         [ 0.0635, -0.3399,  0.8631,  ..., -1.5442, -0.2272, -0.8889],\n",
       "         [-0.7783,  0.4008, -1.1185,  ...,  1.0344, -0.1838,  1.0786]],\n",
       "        requires_grad=True))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake_tensor @ weights + bias, weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nn.Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "            nn.Conv2d(1,20,5),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(20,64,5),\n",
    "            nn.ReLU()\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "            nn.LSTMCell(1, 5),\n",
    "            nn.LSTMCell(1, 5)\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nn.Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.weights = nn.Parameter(torch.randn(784, 10) / math.sqrt(784))\n",
    "        self.bias = nn.Parameter(torch.zeros(10))\n",
    "\n",
    "    def forward(self, xb):\n",
    "        return xb @ self.weights + self.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, num_in, num_out):\n",
    "        super().__init__()\n",
    "        hidden_layer = nn.Linear(num_in, num_out, bias=True)\n",
    "    \n",
    "    def forward(self, input_vector):\n",
    "        return hidden_layer(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self, num_channels, hidden_dim, num_layers=5, target_size=512):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.num_channels = num_channels\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.lstm = nn.LSTM(num_channels, hidden_dim, num_layers=num_layers, batch_first=False)\n",
    "\n",
    "        # The linear layer that maps from hidden state space to tag space\n",
    "        self.hidden2target = nn.Linear(hidden_dim, target_size)\n",
    "\n",
    "    def forward(self, time_stamp):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            time_stamp: (num_feature, batch_size, num_channels)\n",
    "        Output:\n",
    "            metric: (target_size, batch_size)\n",
    "        \"\"\"\n",
    "        # h.shape = c.shape = (num_layers, num_feature, hidden_dim)\n",
    "        #                                     num_feature, batch_size, dim_vector\n",
    "        _, (h, c) = self.lstm(time_stamp.view(-1, len(time_stamp), self.num_channels).double())\n",
    "\n",
    "        # input the last h and c, whose shape = (num_feature, hidden_dim)\n",
    "        # shape = (2 * num_feature, hidden_dim) after concatenation\n",
    "        state_vector = torch.cat((h[-1, :, :], c[-1, :, :]), 0)\n",
    "        state_metric = self.hidden2target(state_vector)\n",
    "        metric = torch.sum(state_metric, dim=0)\n",
    "        return metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lstm = LSTM(1, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM(\n",
      "  (lstm): LSTM(1, 10, num_layers=5)\n",
      "  (hidden2target): Linear(in_features=10, out_features=512, bias=True)\n",
      ")\n",
      "LSTM(1, 10, num_layers=5)\n",
      "Linear(in_features=10, out_features=512, bias=True)\n"
     ]
    }
   ],
   "source": [
    "for mod in lstm.modules():\n",
    "    print(mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM(1, 10, num_layers=5)\n",
      "Linear(in_features=10, out_features=512, bias=True)\n"
     ]
    }
   ],
   "source": [
    "for chd in lstm.children():\n",
    "    print(chd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = nn.Sequential(nn.Linear(2,2), \n",
    "                  nn.ReLU(),\n",
    "                  nn.Sequential(nn.Sigmoid(), nn.ReLU()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear(in_features=2, out_features=2, bias=True)\n",
      "ReLU()\n",
      "Sequential(\n",
      "  (0): Sigmoid()\n",
      "  (1): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "for chd in m.children():\n",
    "    print(chd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=2, out_features=2, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Sequential(\n",
      "    (0): Sigmoid()\n",
      "    (1): ReLU()\n",
      "  )\n",
      ")\n",
      "Linear(in_features=2, out_features=2, bias=True)\n",
      "ReLU()\n",
      "Sequential(\n",
      "  (0): Sigmoid()\n",
      "  (1): ReLU()\n",
      ")\n",
      "Sigmoid()\n",
      "ReLU()\n"
     ]
    }
   ],
   "source": [
    "for mod in m.modules():\n",
    "    print(mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM(\n",
       "  (lstm): LSTM(1, 10, num_layers=5)\n",
       "  (hidden2target): Linear(in_features=10, out_features=512, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(LSTM(1, 10, num_layers=5),\n",
       " Linear(in_features=10, out_features=512, bias=True))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm.lstm, lstm.hidden2target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['lstm.weight_ih_l0', 'lstm.weight_hh_l0', 'lstm.bias_ih_l0', 'lstm.bias_hh_l0', 'lstm.weight_ih_l1', 'lstm.weight_hh_l1', 'lstm.bias_ih_l1', 'lstm.bias_hh_l1', 'lstm.weight_ih_l2', 'lstm.weight_hh_l2', 'lstm.bias_ih_l2', 'lstm.bias_hh_l2', 'lstm.weight_ih_l3', 'lstm.weight_hh_l3', 'lstm.bias_ih_l3', 'lstm.bias_hh_l3', 'lstm.weight_ih_l4', 'lstm.weight_hh_l4', 'lstm.bias_ih_l4', 'lstm.bias_hh_l4', 'hidden2target.weight', 'hidden2target.bias'])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm.state_dict().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0990],\n",
       "        [-0.2571],\n",
       "        [-0.0347],\n",
       "        [-0.0636],\n",
       "        [ 0.2415],\n",
       "        [-0.0642],\n",
       "        [ 0.2453],\n",
       "        [-0.0210],\n",
       "        [ 0.2082],\n",
       "        [-0.2716],\n",
       "        [ 0.1293],\n",
       "        [-0.2409],\n",
       "        [-0.2310],\n",
       "        [ 0.0069],\n",
       "        [ 0.1756],\n",
       "        [ 0.0419],\n",
       "        [ 0.2629],\n",
       "        [-0.2821],\n",
       "        [-0.0298],\n",
       "        [ 0.2047],\n",
       "        [-0.1811],\n",
       "        [ 0.2435],\n",
       "        [-0.0887],\n",
       "        [ 0.0451],\n",
       "        [ 0.0273],\n",
       "        [-0.2344],\n",
       "        [ 0.2546],\n",
       "        [-0.1128],\n",
       "        [ 0.0912],\n",
       "        [-0.2527],\n",
       "        [-0.0728],\n",
       "        [-0.3118],\n",
       "        [ 0.2609],\n",
       "        [-0.0022],\n",
       "        [-0.2772],\n",
       "        [ 0.2821],\n",
       "        [ 0.2772],\n",
       "        [-0.2560],\n",
       "        [-0.1240],\n",
       "        [-0.2942]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm.lstm.weight_ih_l0.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "400\n",
      "40\n",
      "40\n",
      "400\n",
      "400\n",
      "40\n",
      "40\n",
      "400\n",
      "400\n",
      "40\n",
      "40\n",
      "400\n",
      "400\n",
      "40\n",
      "40\n",
      "400\n",
      "400\n",
      "40\n",
      "40\n",
      "5120\n",
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "9672"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pytorch_total_params = 0\n",
    "for p in lstm.parameters():\n",
    "    print(p.numel())\n",
    "    pytorch_total_params += p.numel() if p.requires_grad else 0\n",
    "pytorch_total_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "_lstm = nn.LSTM(input_size=1, hidden_size=5, num_layers=3).double()\n",
    "t = torch.tensor(np_time_stamp[0, :]).view(-1, 1, 1)\n",
    "pred = _lstm(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "100\n",
      "20\n",
      "20\n",
      "100\n",
      "100\n",
      "20\n",
      "20\n",
      "100\n",
      "100\n",
      "20\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "pytorch_total_params = 0\n",
    "for p in _lstm.parameters():\n",
    "    print(p.numel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cell2Lstm(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(Cell2Lstm, self).__init__()\n",
    "        self.seq_layer = torch.nn.Sequential()\n",
    "        self.seq_layer.add_module(\"cell1\", nn.LSTMCell(input_size=input_size, hidden_size=hidden_size))\n",
    "        self.seq_layer.add_module(\"cell2\", nn.LSTMCell(input_size=input_size * 5, hidden_size=hidden_size))\n",
    "        self.seq_layer.add_module(\"cell3\", nn.LSTMCell(input_size=input_size * 5, hidden_size=hidden_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "100\n",
      "20\n",
      "20\n",
      "100\n",
      "100\n",
      "20\n",
      "20\n",
      "100\n",
      "100\n",
      "20\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "cell2lstm = Cell2Lstm(1, 5)\n",
    "for p in cell2lstm.parameters():\n",
    "    print(p.numel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cell2Lstm(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layer):\n",
    "        super(Cell2Lstm, self).__init__()\n",
    "        self.seq_layer = torch.nn.Sequential()\n",
    "        for i in range(num_layer):\n",
    "            i_s = input_size * hidden_size if i > 0 else input_size\n",
    "            self.seq_layer.add_module(\"cell{}\".format(i),\n",
    "                                      nn.LSTMCell(input_size=i_s, hidden_size=hidden_size, bias=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.2071],\n",
      "        [ 0.2067],\n",
      "        [ 0.3792],\n",
      "        [-0.2371],\n",
      "        [ 0.4100],\n",
      "        [ 0.2919],\n",
      "        [-0.2507],\n",
      "        [-0.3056],\n",
      "        [ 0.1491],\n",
      "        [ 0.4270],\n",
      "        [ 0.2330],\n",
      "        [-0.0349],\n",
      "        [-0.3124],\n",
      "        [ 0.0942],\n",
      "        [ 0.0467],\n",
      "        [-0.4152],\n",
      "        [-0.2260],\n",
      "        [ 0.1430],\n",
      "        [ 0.3304],\n",
      "        [ 0.0527]], requires_grad=True) 20\n",
      "Parameter containing:\n",
      "tensor([[-0.1925,  0.1680,  0.3851,  0.1182, -0.1571],\n",
      "        [-0.2092, -0.2862,  0.0501, -0.0069, -0.1397],\n",
      "        [-0.3955,  0.3161,  0.2232, -0.2489,  0.0062],\n",
      "        [-0.4464,  0.0494,  0.1768,  0.4391,  0.2524],\n",
      "        [-0.1627,  0.2647, -0.1970, -0.3628,  0.1867],\n",
      "        [-0.3138, -0.4116, -0.0631, -0.1843,  0.3224],\n",
      "        [ 0.2693, -0.0153, -0.0316,  0.3335, -0.2768],\n",
      "        [ 0.2459,  0.0139,  0.2980, -0.2892, -0.1297],\n",
      "        [ 0.1608, -0.0476, -0.4238,  0.3639, -0.0797],\n",
      "        [ 0.3929,  0.2959,  0.2273,  0.1504,  0.2143],\n",
      "        [-0.1248, -0.3883,  0.3031, -0.2681,  0.0755],\n",
      "        [ 0.0438, -0.4020,  0.1806, -0.2960, -0.1078],\n",
      "        [ 0.1022, -0.2091, -0.0517, -0.0430, -0.1970],\n",
      "        [-0.1927, -0.3771, -0.2404, -0.2592, -0.0633],\n",
      "        [ 0.0641, -0.0276,  0.3089, -0.2486,  0.4237],\n",
      "        [-0.4366,  0.2440, -0.3356,  0.2798,  0.4148],\n",
      "        [ 0.1753, -0.0667, -0.4215, -0.1172,  0.1133],\n",
      "        [-0.3571,  0.2375,  0.3626, -0.3548, -0.3520],\n",
      "        [ 0.0096,  0.0555, -0.1381, -0.3047, -0.0712],\n",
      "        [ 0.2839, -0.3391, -0.1925, -0.1515,  0.1619]], requires_grad=True) 100\n"
     ]
    }
   ],
   "source": [
    "cell2lstm = Cell2Lstm(1, 5, 1)\n",
    "for p in cell2lstm.parameters():\n",
    "    print(p, p.numel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveCustomLSTM(nn.Module):\n",
    "    def __init__(self, input_sz: int, hidden_sz: int):\n",
    "        super().__init__()\n",
    "        self.input_size = input_sz\n",
    "        self.hidden_size = hidden_sz\n",
    "        \n",
    "        #i_t\n",
    "        self.U_i = nn.Parameter(torch.Tensor(input_sz, hidden_sz))\n",
    "        self.V_i = nn.Parameter(torch.Tensor(hidden_sz, hidden_sz))\n",
    "        self.b_i = nn.Parameter(torch.Tensor(hidden_sz))\n",
    "        \n",
    "        #f_t\n",
    "        self.U_f = nn.Parameter(torch.Tensor(input_sz, hidden_sz))\n",
    "        self.V_f = nn.Parameter(torch.Tensor(hidden_sz, hidden_sz))\n",
    "        self.b_f = nn.Parameter(torch.Tensor(hidden_sz))\n",
    "        \n",
    "        #c_t\n",
    "        self.U_c = nn.Parameter(torch.Tensor(input_sz, hidden_sz))\n",
    "        self.V_c = nn.Parameter(torch.Tensor(hidden_sz, hidden_sz))\n",
    "        self.b_c = nn.Parameter(torch.Tensor(hidden_sz))\n",
    "        \n",
    "        #o_t\n",
    "        self.U_o = nn.Parameter(torch.Tensor(input_sz, hidden_sz))\n",
    "        self.V_o = nn.Parameter(torch.Tensor(hidden_sz, hidden_sz))\n",
    "        self.b_o = nn.Parameter(torch.Tensor(hidden_sz))\n",
    "        \n",
    "    def forward(self, x, init_states=None):\n",
    "            bs, seq_sz, _ = x.size()\n",
    "            hidden_seq = []\n",
    "\n",
    "            if init_states is None:\n",
    "                h_t, c_t = (\n",
    "                    torch.zeros(bs, self.hidden_size).to(x.device),\n",
    "                    torch.zeros(bs, self.hidden_size).to(x.device),\n",
    "                )\n",
    "            else:\n",
    "                h_t, c_t = init_states\n",
    "\n",
    "            for t in range(seq_sz):\n",
    "                x_t = x[:, t, :]\n",
    "\n",
    "                i_t = torch.sigmoid(x_t @ self.U_i + h_t @ self.V_i + self.b_i)\n",
    "                f_t = torch.sigmoid(x_t @ self.U_f + h_t @ self.V_f + self.b_f)\n",
    "                g_t = torch.tanh(x_t @ self.U_c + h_t @ self.V_c + self.b_c)\n",
    "                o_t = torch.sigmoid(x_t @ self.U_o + h_t @ self.V_o + self.b_o)\n",
    "                c_t = f_t * c_t + i_t * g_t\n",
    "                h_t = o_t * torch.tanh(c_t)\n",
    "\n",
    "                hidden_seq.append(h_t.unsqueeze(0))\n",
    "\n",
    "            #reshape hidden_seq p/ retornar\n",
    "            hidden_seq = torch.cat(hidden_seq, dim=0)\n",
    "            hidden_seq = hidden_seq.transpose(0, 1).contiguous()\n",
    "            return hidden_seq, (h_t, c_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "25\n",
      "5\n",
      "5\n",
      "25\n",
      "5\n",
      "5\n",
      "25\n",
      "5\n",
      "5\n",
      "25\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "nc_lstm = NaiveCustomLSTM(1, 5)\n",
    "for p in nc_lstm.parameters():\n",
    "    print(p.numel())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modulelist and ModuleDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearNet(nn.Module):\n",
    "    def __init__(self, input_size, num_layers, layers_size, output_size):\n",
    "        super(LinearNet, self).__init__()\n",
    "\n",
    "        self.linears = nn.ModuleList([nn.Linear(input_size, layers_size)])\n",
    "        self.linears.extend([nn.Linear(layers_size, layers_size) for i in range(1, self.num_layers-1)])\n",
    "        # self.linears.append(nn.Linear(layers_size, output_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlexNet(nn.Module):\n",
    "    def __init__(self, num_classes=1000):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features =  nn.ModuleDict({\n",
    "            \"Conv2d_1\":nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            \"BN_1\":nn.BatchNorm2d(64),\n",
    "            \"Act_1\":nn.ReLU(inplace=True),\n",
    "            \"Max_1\":nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "\n",
    "            \"Conv2d_2\":nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            \"BN_2\":nn.BatchNorm2d(192),\n",
    "            \"Act_2\":nn.ReLU(inplace=True),\n",
    "            \"Max_2\":nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "\n",
    "            \"Conv2d_3\":nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            \"BN_3\":nn.BatchNorm2d(384),\n",
    "            \"Act_3\":nn.ReLU(inplace=True),\n",
    "            \n",
    "            \"Conv2d_4\":nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            \"BN_4\":nn.BatchNorm2d(64),\n",
    "            \"Act_4\":nn.ReLU(inplace=True),\n",
    "            \n",
    "            \"Conv2d_5\":nn.Conv2d(256, 256, kernel_size=3, padding=2),\n",
    "            \"BN_5\":nn.BatchNorm2d(64),\n",
    "            \"Act_5\":nn.ReLU(inplace=True),\n",
    "            \"Max_5\":nn.MaxPool2d(kernel_size=3, stride=2)\n",
    "        })\n",
    "\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
    "\n",
    "        self.fullyconnected = nn.ModuleDict({\n",
    "            \"Pool\":nn.AdaptiveAvgPool2d((6, 6)),\n",
    "            \"drop_6\":nn.Dropout(),\n",
    "            \"Linear_6\":nn.Linear(256 * 6 * 6, 4096),\n",
    "            #\"BN_6\":nn.BatchNorm1d(4096),\n",
    "            \"Act_6\":nn.ReLU(inplace=True),\n",
    "            \"drop_7\":nn.Dropout(),\n",
    "            \"Linear_7\":nn.Linear(4096, 4096),\n",
    "            #\"BN_7\":nn.BatchNorm1d(4096),\n",
    "            \"Act_7\":nn.ReLU(inplace=True),\n",
    "            \"Linear_8\":nn.Linear(4096, num_classes),\n",
    "            #\"BN_8\":nn.BatchNorm1d(num_classes),\n",
    "            #\"Softmax\":nn.LogSoftmax()\n",
    "        })\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features['Conv2d_1'](x)\n",
    "        x = self.features['Act_1'](x)\n",
    "        x = self.features['Max_1'](x)\n",
    "        x = self.features['Conv2d_2'](x)\n",
    "        x = self.features['Act_2'](x)\n",
    "        x = self.features['Max_2'](x)\n",
    "        x = self.features['Conv2d_3'](x)\n",
    "        x = self.features['Act_3'](x)\n",
    "        x = self.features['Conv2d_4'](x)\n",
    "        x = self.features['Act_4'](x)\n",
    "        x = self.features['Conv2d_5'](x)\n",
    "        x = self.features['Act_5'](x)\n",
    "        x = self.features['Max_5'](x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(-1, 256 * 6 * 6)\n",
    "        x = self.fullyconnected['Linear_6'](x)\n",
    "        x = self.fullyconnected['Act_6'](x)\n",
    "        x = self.fullyconnected['Linear_7'](x)\n",
    "        x = self.fullyconnected['Act_7'](x)\n",
    "        x = self.fullyconnected['Linear_8'](x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}