multi_model_testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#From mnist example in pytorch example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "import torch.utils.data\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train( model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.nll_loss(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item()))\n",
    "\n",
    "def test( model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss\n",
    "            pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "batch_size = 64\n",
    "#kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
    "train_dataset =  datasets.MNIST('../data', train=True, transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ]))\n",
    "test_dataset =  datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ]))\n",
    "\n",
    "model1_train = torch.utils.data.Subset(train_dataset,range(25000))\n",
    "model2_train = torch.utils.data.Subset(train_dataset,range(25000,50000))\n",
    "combine_train = torch.utils.data.Subset(train_dataset,range(50000,60000))\n",
    "\n",
    "model1_train_loader = torch.utils.data.DataLoader(model1_train,batch_size=batch_size, shuffle=True)\n",
    "model2_train_loader = torch.utils.data.DataLoader(model2_train,batch_size=batch_size, shuffle=True)\n",
    "combine_train_loader = torch.utils.data.DataLoader(combine_train,batch_size=batch_size, shuffle=True)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset,batch_size=batch_size, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.376938\n",
      "Train Epoch: 1 [6400/60000 (11%)]\tLoss: 0.903542\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.632314\n",
      "Train Epoch: 1 [19200/60000 (32%)]\tLoss: 0.457679\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.443795\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.417793\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.360744\n",
      "Train Epoch: 1 [44800/60000 (75%)]\tLoss: 0.343218\n",
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.316730\n",
      "Train Epoch: 1 [57600/60000 (96%)]\tLoss: 0.312192\n",
      "\n",
      "Test set: Average loss: 0.1212, Accuracy: 9625/10000 (96%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.309151\n",
      "Train Epoch: 2 [6400/60000 (11%)]\tLoss: 0.404300\n",
      "Train Epoch: 2 [12800/60000 (21%)]\tLoss: 0.493842\n",
      "Train Epoch: 2 [19200/60000 (32%)]\tLoss: 0.206859\n",
      "Train Epoch: 2 [25600/60000 (43%)]\tLoss: 0.271278\n",
      "Train Epoch: 2 [32000/60000 (53%)]\tLoss: 0.182909\n",
      "Train Epoch: 2 [38400/60000 (64%)]\tLoss: 0.306855\n",
      "Train Epoch: 2 [44800/60000 (75%)]\tLoss: 0.350212\n",
      "Train Epoch: 2 [51200/60000 (85%)]\tLoss: 0.255203\n",
      "Train Epoch: 2 [57600/60000 (96%)]\tLoss: 0.177814\n",
      "\n",
      "Test set: Average loss: 0.0810, Accuracy: 9750/10000 (98%)\n",
      "\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.317871\n",
      "Train Epoch: 3 [6400/60000 (11%)]\tLoss: 0.115484\n",
      "Train Epoch: 3 [12800/60000 (21%)]\tLoss: 0.204299\n",
      "Train Epoch: 3 [19200/60000 (32%)]\tLoss: 0.287520\n",
      "Train Epoch: 3 [25600/60000 (43%)]\tLoss: 0.120748\n",
      "Train Epoch: 3 [32000/60000 (53%)]\tLoss: 0.201726\n",
      "Train Epoch: 3 [38400/60000 (64%)]\tLoss: 0.258438\n",
      "Train Epoch: 3 [44800/60000 (75%)]\tLoss: 0.068262\n",
      "Train Epoch: 3 [51200/60000 (85%)]\tLoss: 0.366711\n",
      "Train Epoch: 3 [57600/60000 (96%)]\tLoss: 0.193047\n",
      "\n",
      "Test set: Average loss: 0.0586, Accuracy: 9826/10000 (98%)\n",
      "\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.193078\n",
      "Train Epoch: 4 [6400/60000 (11%)]\tLoss: 0.163633\n",
      "Train Epoch: 4 [12800/60000 (21%)]\tLoss: 0.255491\n",
      "Train Epoch: 4 [19200/60000 (32%)]\tLoss: 0.231564\n",
      "Train Epoch: 4 [25600/60000 (43%)]\tLoss: 0.111181\n",
      "Train Epoch: 4 [32000/60000 (53%)]\tLoss: 0.154760\n",
      "Train Epoch: 4 [38400/60000 (64%)]\tLoss: 0.134769\n",
      "Train Epoch: 4 [44800/60000 (75%)]\tLoss: 0.182664\n",
      "Train Epoch: 4 [51200/60000 (85%)]\tLoss: 0.246271\n",
      "Train Epoch: 4 [57600/60000 (96%)]\tLoss: 0.143767\n",
      "\n",
      "Test set: Average loss: 0.0531, Accuracy: 9832/10000 (98%)\n",
      "\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.267769\n",
      "Train Epoch: 5 [6400/60000 (11%)]\tLoss: 0.292296\n",
      "Train Epoch: 5 [12800/60000 (21%)]\tLoss: 0.202063\n",
      "Train Epoch: 5 [19200/60000 (32%)]\tLoss: 0.155944\n",
      "Train Epoch: 5 [25600/60000 (43%)]\tLoss: 0.230029\n",
      "Train Epoch: 5 [32000/60000 (53%)]\tLoss: 0.194438\n",
      "Train Epoch: 5 [38400/60000 (64%)]\tLoss: 0.120719\n",
      "Train Epoch: 5 [44800/60000 (75%)]\tLoss: 0.303011\n",
      "Train Epoch: 5 [51200/60000 (85%)]\tLoss: 0.149667\n",
      "Train Epoch: 5 [57600/60000 (96%)]\tLoss: 0.130209\n",
      "\n",
      "Test set: Average loss: 0.0486, Accuracy: 9849/10000 (98%)\n",
      "\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.199312\n",
      "Train Epoch: 6 [6400/60000 (11%)]\tLoss: 0.183375\n",
      "Train Epoch: 6 [12800/60000 (21%)]\tLoss: 0.093987\n",
      "Train Epoch: 6 [19200/60000 (32%)]\tLoss: 0.061531\n",
      "Train Epoch: 6 [25600/60000 (43%)]\tLoss: 0.094693\n",
      "Train Epoch: 6 [32000/60000 (53%)]\tLoss: 0.117702\n",
      "Train Epoch: 6 [38400/60000 (64%)]\tLoss: 0.108946\n",
      "Train Epoch: 6 [44800/60000 (75%)]\tLoss: 0.218418\n",
      "Train Epoch: 6 [51200/60000 (85%)]\tLoss: 0.194026\n",
      "Train Epoch: 6 [57600/60000 (96%)]\tLoss: 0.391915\n",
      "\n",
      "Test set: Average loss: 0.0468, Accuracy: 9849/10000 (98%)\n",
      "\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.126935\n",
      "Train Epoch: 7 [6400/60000 (11%)]\tLoss: 0.111194\n",
      "Train Epoch: 7 [12800/60000 (21%)]\tLoss: 0.070316\n",
      "Train Epoch: 7 [19200/60000 (32%)]\tLoss: 0.063177\n",
      "Train Epoch: 7 [25600/60000 (43%)]\tLoss: 0.030420\n",
      "Train Epoch: 7 [32000/60000 (53%)]\tLoss: 0.215397\n",
      "Train Epoch: 7 [38400/60000 (64%)]\tLoss: 0.282832\n",
      "Train Epoch: 7 [44800/60000 (75%)]\tLoss: 0.218782\n",
      "Train Epoch: 7 [51200/60000 (85%)]\tLoss: 0.112336\n",
      "Train Epoch: 7 [57600/60000 (96%)]\tLoss: 0.390633\n",
      "\n",
      "Test set: Average loss: 0.0437, Accuracy: 9861/10000 (99%)\n",
      "\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.161898\n",
      "Train Epoch: 8 [6400/60000 (11%)]\tLoss: 0.138241\n",
      "Train Epoch: 8 [12800/60000 (21%)]\tLoss: 0.114721\n",
      "Train Epoch: 8 [19200/60000 (32%)]\tLoss: 0.173094\n",
      "Train Epoch: 8 [25600/60000 (43%)]\tLoss: 0.159061\n",
      "Train Epoch: 8 [32000/60000 (53%)]\tLoss: 0.107298\n",
      "Train Epoch: 8 [38400/60000 (64%)]\tLoss: 0.165657\n",
      "Train Epoch: 8 [44800/60000 (75%)]\tLoss: 0.093654\n",
      "Train Epoch: 8 [51200/60000 (85%)]\tLoss: 0.178623\n",
      "Train Epoch: 8 [57600/60000 (96%)]\tLoss: 0.170279\n",
      "\n",
      "Test set: Average loss: 0.0424, Accuracy: 9880/10000 (99%)\n",
      "\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.090449\n",
      "Train Epoch: 9 [6400/60000 (11%)]\tLoss: 0.183217\n",
      "Train Epoch: 9 [12800/60000 (21%)]\tLoss: 0.055900\n",
      "Train Epoch: 9 [19200/60000 (32%)]\tLoss: 0.125606\n",
      "Train Epoch: 9 [25600/60000 (43%)]\tLoss: 0.210432\n",
      "Train Epoch: 9 [32000/60000 (53%)]\tLoss: 0.068196\n",
      "Train Epoch: 9 [38400/60000 (64%)]\tLoss: 0.140097\n",
      "Train Epoch: 9 [44800/60000 (75%)]\tLoss: 0.113998\n",
      "Train Epoch: 9 [51200/60000 (85%)]\tLoss: 0.134533\n",
      "Train Epoch: 9 [57600/60000 (96%)]\tLoss: 0.060074\n",
      "\n",
      "Test set: Average loss: 0.0403, Accuracy: 9883/10000 (99%)\n",
      "\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.153709\n",
      "Train Epoch: 10 [6400/60000 (11%)]\tLoss: 0.090462\n",
      "Train Epoch: 10 [12800/60000 (21%)]\tLoss: 0.318536\n",
      "Train Epoch: 10 [19200/60000 (32%)]\tLoss: 0.347748\n",
      "Train Epoch: 10 [25600/60000 (43%)]\tLoss: 0.169369\n",
      "Train Epoch: 10 [32000/60000 (53%)]\tLoss: 0.135781\n",
      "Train Epoch: 10 [38400/60000 (64%)]\tLoss: 0.167947\n",
      "Train Epoch: 10 [44800/60000 (75%)]\tLoss: 0.190426\n",
      "Train Epoch: 10 [51200/60000 (85%)]\tLoss: 0.144162\n",
      "Train Epoch: 10 [57600/60000 (96%)]\tLoss: 0.139054\n",
      "\n",
      "Test set: Average loss: 0.0379, Accuracy: 9887/10000 (99%)\n",
      "\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.073730\n",
      "Train Epoch: 11 [6400/60000 (11%)]\tLoss: 0.019118\n",
      "Train Epoch: 11 [12800/60000 (21%)]\tLoss: 0.049790\n",
      "Train Epoch: 11 [19200/60000 (32%)]\tLoss: 0.038488\n",
      "Train Epoch: 11 [25600/60000 (43%)]\tLoss: 0.125378\n",
      "Train Epoch: 11 [32000/60000 (53%)]\tLoss: 0.212092\n",
      "Train Epoch: 11 [38400/60000 (64%)]\tLoss: 0.148884\n",
      "Train Epoch: 11 [44800/60000 (75%)]\tLoss: 0.137312\n",
      "Train Epoch: 11 [51200/60000 (85%)]\tLoss: 0.290896\n",
      "Train Epoch: 11 [57600/60000 (96%)]\tLoss: 0.282631\n",
      "\n",
      "Test set: Average loss: 0.0406, Accuracy: 9882/10000 (99%)\n",
      "\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.083473\n",
      "Train Epoch: 12 [6400/60000 (11%)]\tLoss: 0.433820\n",
      "Train Epoch: 12 [12800/60000 (21%)]\tLoss: 0.122699\n",
      "Train Epoch: 12 [19200/60000 (32%)]\tLoss: 0.120592\n",
      "Train Epoch: 12 [25600/60000 (43%)]\tLoss: 0.110529\n",
      "Train Epoch: 12 [32000/60000 (53%)]\tLoss: 0.118363\n",
      "Train Epoch: 12 [38400/60000 (64%)]\tLoss: 0.059135\n",
      "Train Epoch: 12 [44800/60000 (75%)]\tLoss: 0.130841\n",
      "Train Epoch: 12 [51200/60000 (85%)]\tLoss: 0.098280\n",
      "Train Epoch: 12 [57600/60000 (96%)]\tLoss: 0.257019\n",
      "\n",
      "Test set: Average loss: 0.0381, Accuracy: 9877/10000 (99%)\n",
      "\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.055712\n",
      "Train Epoch: 13 [6400/60000 (11%)]\tLoss: 0.152188\n",
      "Train Epoch: 13 [12800/60000 (21%)]\tLoss: 0.078694\n",
      "Train Epoch: 13 [19200/60000 (32%)]\tLoss: 0.084603\n",
      "Train Epoch: 13 [25600/60000 (43%)]\tLoss: 0.123667\n",
      "Train Epoch: 13 [32000/60000 (53%)]\tLoss: 0.057518\n",
      "Train Epoch: 13 [38400/60000 (64%)]\tLoss: 0.064377\n",
      "Train Epoch: 13 [44800/60000 (75%)]\tLoss: 0.280156\n",
      "Train Epoch: 13 [51200/60000 (85%)]\tLoss: 0.035858\n",
      "Train Epoch: 13 [57600/60000 (96%)]\tLoss: 0.277488\n",
      "\n",
      "Test set: Average loss: 0.0362, Accuracy: 9894/10000 (99%)\n",
      "\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.102422\n",
      "Train Epoch: 14 [6400/60000 (11%)]\tLoss: 0.052921\n",
      "Train Epoch: 14 [12800/60000 (21%)]\tLoss: 0.092955\n",
      "Train Epoch: 14 [19200/60000 (32%)]\tLoss: 0.131674\n",
      "Train Epoch: 14 [25600/60000 (43%)]\tLoss: 0.123248\n",
      "Train Epoch: 14 [32000/60000 (53%)]\tLoss: 0.098639\n",
      "Train Epoch: 14 [38400/60000 (64%)]\tLoss: 0.030072\n",
      "Train Epoch: 14 [44800/60000 (75%)]\tLoss: 0.204350\n",
      "Train Epoch: 14 [51200/60000 (85%)]\tLoss: 0.139084\n",
      "Train Epoch: 14 [57600/60000 (96%)]\tLoss: 0.220049\n",
      "\n",
      "Test set: Average loss: 0.0329, Accuracy: 9906/10000 (99%)\n",
      "\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 0.095760\n",
      "Train Epoch: 15 [6400/60000 (11%)]\tLoss: 0.152569\n",
      "Train Epoch: 15 [12800/60000 (21%)]\tLoss: 0.084596\n",
      "Train Epoch: 15 [19200/60000 (32%)]\tLoss: 0.138875\n",
      "Train Epoch: 15 [25600/60000 (43%)]\tLoss: 0.153610\n",
      "Train Epoch: 15 [32000/60000 (53%)]\tLoss: 0.155402\n",
      "Train Epoch: 15 [38400/60000 (64%)]\tLoss: 0.093098\n",
      "Train Epoch: 15 [44800/60000 (75%)]\tLoss: 0.088751\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 15 [51200/60000 (85%)]\tLoss: 0.135983\n",
      "Train Epoch: 15 [57600/60000 (96%)]\tLoss: 0.051386\n",
      "\n",
      "Test set: Average loss: 0.0324, Accuracy: 9897/10000 (99%)\n",
      "\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 0.109488\n",
      "Train Epoch: 16 [6400/60000 (11%)]\tLoss: 0.008799\n",
      "Train Epoch: 16 [12800/60000 (21%)]\tLoss: 0.136251\n",
      "Train Epoch: 16 [19200/60000 (32%)]\tLoss: 0.060284\n",
      "Train Epoch: 16 [25600/60000 (43%)]\tLoss: 0.053560\n",
      "Train Epoch: 16 [32000/60000 (53%)]\tLoss: 0.061247\n",
      "Train Epoch: 16 [38400/60000 (64%)]\tLoss: 0.124663\n",
      "Train Epoch: 16 [44800/60000 (75%)]\tLoss: 0.382084\n",
      "Train Epoch: 16 [51200/60000 (85%)]\tLoss: 0.172176\n",
      "Train Epoch: 16 [57600/60000 (96%)]\tLoss: 0.085447\n",
      "\n",
      "Test set: Average loss: 0.0332, Accuracy: 9893/10000 (99%)\n",
      "\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 0.055696\n",
      "Train Epoch: 17 [6400/60000 (11%)]\tLoss: 0.262721\n",
      "Train Epoch: 17 [12800/60000 (21%)]\tLoss: 0.047206\n",
      "Train Epoch: 17 [19200/60000 (32%)]\tLoss: 0.093576\n",
      "Train Epoch: 17 [25600/60000 (43%)]\tLoss: 0.056773\n",
      "Train Epoch: 17 [32000/60000 (53%)]\tLoss: 0.130911\n",
      "Train Epoch: 17 [38400/60000 (64%)]\tLoss: 0.144952\n",
      "Train Epoch: 17 [44800/60000 (75%)]\tLoss: 0.188942\n",
      "Train Epoch: 17 [51200/60000 (85%)]\tLoss: 0.157970\n",
      "Train Epoch: 17 [57600/60000 (96%)]\tLoss: 0.094647\n",
      "\n",
      "Test set: Average loss: 0.0343, Accuracy: 9904/10000 (99%)\n",
      "\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 0.029152\n",
      "Train Epoch: 18 [6400/60000 (11%)]\tLoss: 0.021387\n",
      "Train Epoch: 18 [12800/60000 (21%)]\tLoss: 0.215030\n",
      "Train Epoch: 18 [19200/60000 (32%)]\tLoss: 0.170424\n",
      "Train Epoch: 18 [25600/60000 (43%)]\tLoss: 0.083901\n",
      "Train Epoch: 18 [32000/60000 (53%)]\tLoss: 0.208507\n",
      "Train Epoch: 18 [38400/60000 (64%)]\tLoss: 0.142527\n",
      "Train Epoch: 18 [44800/60000 (75%)]\tLoss: 0.086175\n",
      "Train Epoch: 18 [51200/60000 (85%)]\tLoss: 0.128493\n",
      "Train Epoch: 18 [57600/60000 (96%)]\tLoss: 0.357631\n",
      "\n",
      "Test set: Average loss: 0.0301, Accuracy: 9899/10000 (99%)\n",
      "\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 0.036069\n",
      "Train Epoch: 19 [6400/60000 (11%)]\tLoss: 0.371722\n",
      "Train Epoch: 19 [12800/60000 (21%)]\tLoss: 0.131197\n",
      "Train Epoch: 19 [19200/60000 (32%)]\tLoss: 0.260571\n",
      "Train Epoch: 19 [25600/60000 (43%)]\tLoss: 0.021166\n",
      "Train Epoch: 19 [32000/60000 (53%)]\tLoss: 0.063948\n",
      "Train Epoch: 19 [38400/60000 (64%)]\tLoss: 0.246548\n",
      "Train Epoch: 19 [44800/60000 (75%)]\tLoss: 0.199971\n",
      "Train Epoch: 19 [51200/60000 (85%)]\tLoss: 0.100753\n",
      "Train Epoch: 19 [57600/60000 (96%)]\tLoss: 0.171156\n",
      "\n",
      "Test set: Average loss: 0.0312, Accuracy: 9898/10000 (99%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = Net()\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.05)\n",
    "\n",
    "single_model_start_time = time.time()\n",
    "for epoch in range(1, 20):\n",
    "    train( model, device, train_loader, optimizer, epoch)\n",
    "    test( model, device, test_loader)\n",
    "single_model_end_time = time.time()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train both models separately on differnt datasets\n",
    "model1 = Net()\n",
    "model2 = Net()\n",
    "\n",
    "optimizer1 = optim.SGD(model1.parameters(), lr=0.05)\n",
    "\n",
    "model1_start_time = time.time()\n",
    "for epoch in range(1, 20):\n",
    "    train( model1, device, model1_train_loader, optimizer1, epoch)\n",
    "    test( model1, device, test_loader)\n",
    "model1_end_time = time.time()\n",
    "\n",
    "optimizer2 = optim.SGD(model2.parameters(), lr=0.05)\n",
    "model2_start_time = time.time()\n",
    "for epoch in range(1, 20):\n",
    "    train( model2, device, model2_train_loader, optimizer2, epoch)\n",
    "    test( model2, device, test_loader)   \n",
    "model2_end_time = time.time()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/10000 (0%)]\tLoss: 0.388519\n",
      "Train Epoch: 1 [6400/10000 (64%)]\tLoss: 0.438580\n",
      "Train Epoch: 2 [0/10000 (0%)]\tLoss: 0.444665\n",
      "Train Epoch: 2 [6400/10000 (64%)]\tLoss: 0.279676\n",
      "Train Epoch: 3 [0/10000 (0%)]\tLoss: 0.144020\n",
      "Train Epoch: 3 [6400/10000 (64%)]\tLoss: 0.525121\n",
      "Train Epoch: 4 [0/10000 (0%)]\tLoss: 0.310531\n",
      "Train Epoch: 4 [6400/10000 (64%)]\tLoss: 0.403705\n",
      "Train Epoch: 5 [0/10000 (0%)]\tLoss: 0.315960\n",
      "Train Epoch: 5 [6400/10000 (64%)]\tLoss: 0.324131\n",
      "Train Epoch: 6 [0/10000 (0%)]\tLoss: 0.426520\n",
      "Train Epoch: 6 [6400/10000 (64%)]\tLoss: 0.192550\n",
      "Train Epoch: 7 [0/10000 (0%)]\tLoss: 0.347938\n",
      "Train Epoch: 7 [6400/10000 (64%)]\tLoss: 0.424191\n",
      "Train Epoch: 8 [0/10000 (0%)]\tLoss: 0.562397\n",
      "Train Epoch: 8 [6400/10000 (64%)]\tLoss: 0.309457\n",
      "Train Epoch: 9 [0/10000 (0%)]\tLoss: 0.157770\n",
      "Train Epoch: 9 [6400/10000 (64%)]\tLoss: 0.591936\n",
      "Train Epoch: 10 [0/10000 (0%)]\tLoss: 0.573923\n",
      "Train Epoch: 10 [6400/10000 (64%)]\tLoss: 0.202599\n",
      "Train Epoch: 11 [0/10000 (0%)]\tLoss: 0.482513\n",
      "Train Epoch: 11 [6400/10000 (64%)]\tLoss: 0.368853\n",
      "Train Epoch: 12 [0/10000 (0%)]\tLoss: 0.179395\n",
      "Train Epoch: 12 [6400/10000 (64%)]\tLoss: 0.287911\n",
      "Train Epoch: 13 [0/10000 (0%)]\tLoss: 0.230029\n",
      "Train Epoch: 13 [6400/10000 (64%)]\tLoss: 0.124150\n",
      "Train Epoch: 14 [0/10000 (0%)]\tLoss: 0.247308\n",
      "Train Epoch: 14 [6400/10000 (64%)]\tLoss: 0.640382\n",
      "Train Epoch: 15 [0/10000 (0%)]\tLoss: 0.094283\n",
      "Train Epoch: 15 [6400/10000 (64%)]\tLoss: 0.226976\n",
      "Train Epoch: 16 [0/10000 (0%)]\tLoss: 0.269353\n",
      "Train Epoch: 16 [6400/10000 (64%)]\tLoss: 0.389134\n",
      "Train Epoch: 17 [0/10000 (0%)]\tLoss: 0.248906\n",
      "Train Epoch: 17 [6400/10000 (64%)]\tLoss: 0.159057\n",
      "Train Epoch: 18 [0/10000 (0%)]\tLoss: 0.115029\n",
      "Train Epoch: 18 [6400/10000 (64%)]\tLoss: 0.228565\n",
      "Train Epoch: 19 [0/10000 (0%)]\tLoss: 0.236745\n",
      "Train Epoch: 19 [6400/10000 (64%)]\tLoss: 0.145297\n"
     ]
    }
   ],
   "source": [
    "#Combined training\n",
    "combine_train_start_time = time.time()\n",
    "for epoch in range(1, 20):\n",
    "    model1.train()\n",
    "    model2.train()\n",
    "    for batch_idx, (data, target) in enumerate(combine_train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer1.zero_grad()\n",
    "        optimizer2.zero_grad()\n",
    "        output1 = model1(data)\n",
    "        output2 = model2(data)\n",
    "        pred1 = output1.max(1, keepdim=True)[1]\n",
    "        pred2 = output2.max(1, keepdim=True)[1]\n",
    "        loss1 = F.nll_loss(output1, pred2.view(-1))\n",
    "        loss1.backward()\n",
    "        optimizer1.step()\n",
    "        loss2 = F.nll_loss(output2, pred1.view(-1))\n",
    "        loss2.backward()\n",
    "        optimizer2.step()\n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(combine_train_loader.dataset),\n",
    "                100. * batch_idx / len(combine_train_loader), loss1.item(), loss2.item))\n",
    "combine_train_end_time = time.time()            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "839.3560137748718"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "single_model_end_time-single_model_start_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "705.5945563316345"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_end_time-model1_start_time + combine_train_end_time -combine_train_start_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average loss: 0.0736, Accuracy: 9776/10000 (98%)\n",
      "\n",
      "\n",
      "Test set: Average loss: 0.0735, Accuracy: 9772/10000 (98%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test( model2, device, test_loader) \n",
    "test( model1, device, test_loader) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}