09_acitavtion_and_optimizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# for reproducibility\n",
    "np.random.seed(777)\n",
    "torch.manual_seed(777)\n",
    "if device == 'cuda':\n",
    "    torch.cuda.manual_seed_all(777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "lr = 0.001\n",
    "training_epochs = 15\n",
    "batch_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train = torchvision.datasets.MNIST(root='', train=True, transform=transforms.ToTensor())\n",
    "dataset_test = torchvision.datasets.MNIST(root='', train=False, transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, drop_last=True)\n",
    "dataloader_test = DataLoader(dataset_test, batch_size=batch_size, shuffle=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(nn.Linear(28*28, 256), nn.ReLU(),\n",
    "                      nn.Linear(256, 256), nn.ReLU(),\n",
    "                      nn.Linear(256, 10)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in model.children():\n",
    "    if isinstance(x, nn.Linear):\n",
    "        nn.init.normal_(x.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training...\n",
      "epoch 1    training loss:176.040    training acc:72.640    test loss:48.500    test acc:86.610\n",
      "epoch 2    training loss:38.305    training acc:88.550    test loss:31.462    test acc:90.020\n",
      "epoch 3    training loss:24.103    training acc:91.527    test loss:24.760    test acc:91.100\n",
      "epoch 4    training loss:16.790    training acc:93.077    test loss:20.484    test acc:92.250\n",
      "epoch 5    training loss:11.952    training acc:94.418    test loss:18.159    test acc:92.890\n",
      "epoch 6    training loss:8.808    training acc:95.243    test loss:16.572    test acc:93.370\n",
      "epoch 7    training loss:6.505    training acc:95.965    test loss:14.989    test acc:93.740\n",
      "epoch 8    training loss:4.728    training acc:96.728    test loss:13.755    test acc:93.980\n",
      "epoch 9    training loss:3.446    training acc:97.323    test loss:12.740    test acc:94.630\n",
      "epoch 10    training loss:2.591    training acc:97.683    test loss:13.305    test acc:94.260\n",
      "epoch 11    training loss:1.854    training acc:98.125    test loss:12.690    test acc:94.390\n",
      "epoch 12    training loss:1.452    training acc:98.393    test loss:11.990    test acc:94.520\n",
      "epoch 13    training loss:1.084    training acc:98.658    test loss:12.018    test acc:94.880\n",
      "epoch 14    training loss:0.986    training acc:98.757    test loss:11.936    test acc:94.660\n",
      "epoch 15    training loss:0.788    training acc:98.847    test loss:11.574    test acc:95.010\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "print('Start training...')\n",
    "\n",
    "for epoch in range(1, training_epochs+1):\n",
    "\n",
    "    running_loss = 0\n",
    "    running_correct = 0\n",
    "\n",
    "    for X, y in dataloader_train:\n",
    "        X, y = X.view(-1,28*28).to(device), y.to(device)\n",
    "        y_pred = model(X)\n",
    "        loss = criterion(y_pred, y)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        with torch.no_grad():\n",
    "            running_loss += loss.item()\n",
    "            running_correct += sum(y_pred.argmax(dim=1) == y)\n",
    "\n",
    "    running_test_loss = 0\n",
    "    running_test_correct = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader_test:\n",
    "            X, y = X.view(-1,28*28).to(device), y.to(device)\n",
    "            y_pred = model(X)\n",
    "            loss = criterion(y_pred, y)\n",
    "\n",
    "            running_test_loss += loss.item()\n",
    "            running_test_correct += sum(y_pred.argmax(dim=1) == y)\n",
    "    \n",
    "    print(f'epoch {epoch}\\\n",
    "    training loss:{running_loss/len(dataloader_train):.3f}\\\n",
    "    training acc:{running_correct/len(dataloader_train):.3f}\\\n",
    "    test loss:{running_test_loss/len(dataloader_test):.3f}\\\n",
    "    test acc:{running_test_correct/len(dataloader_test):.3f}')\n",
    "\n",
    "print('Finished Training')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}