4-2_Knowledge_Distillation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 先训练老师网络 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms\n",
    "import torch.utils.data\n",
    "\n",
    "\n",
    "torch.manual_seed(0)\n",
    "torch.cuda.manual_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TeacherNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(TeacherNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 32, 3, 1)\n",
    "        self.conv2 = nn.Conv2d(32, 64, 3, 1)\n",
    "        self.dropout1 = nn.Dropout2d(0.3)\n",
    "        self.dropout2 = nn.Dropout2d(0.5)\n",
    "        self.fc1 = nn.Linear(9216, 128)\n",
    "        self.fc2 = nn.Linear(128, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, 2)\n",
    "        x = self.dropout1(x)\n",
    "        x = torch.flatten(x, 1)\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.dropout2(x)\n",
    "        output = self.fc2(x)\n",
    "        return output\n",
    "\n",
    "\n",
    "def train_teacher(model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    trained_samples = 0\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        trained_samples += len(data)\n",
    "        progress = math.ceil(batch_idx / len(train_loader) * 50)\n",
    "        print(\"\\rTrain epoch %d: %d/%d, [%-51s] %d%%\" %\n",
    "              (epoch, trained_samples, len(train_loader.dataset),\n",
    "               '-' * progress + '>', progress * 2), end='')\n",
    "\n",
    "\n",
    "def test_teacher(model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.cross_entropy(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest: average loss: {:.4f}, accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "    return test_loss, correct / len(test_loader.dataset)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def teacher_main():\n",
    "    epochs = 10\n",
    "    batch_size = 64\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=False, download=True, transform=transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,))\n",
    "        ])),\n",
    "        batch_size=1000, shuffle=True)\n",
    "\n",
    "    model = TeacherNet().to(device)\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "    \n",
    "    teacher_history = []\n",
    "\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        train_teacher(model, device, train_loader, optimizer, epoch)\n",
    "        loss, acc = test_teacher(model, device, test_loader)\n",
    "        \n",
    "        teacher_history.append((loss, acc))\n",
    "\n",
    "    torch.save(model.state_dict(), \"teacher.pt\")\n",
    "    return model, teacher_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 1: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0510, accuracy: 9832/10000 (98%)\n",
      "Train epoch 2: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0364, accuracy: 9882/10000 (99%)\n",
      "Train epoch 3: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0354, accuracy: 9883/10000 (99%)\n",
      "Train epoch 4: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0325, accuracy: 9902/10000 (99%)\n",
      "Train epoch 5: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0314, accuracy: 9897/10000 (99%)\n",
      "Train epoch 6: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0330, accuracy: 9898/10000 (99%)\n",
      "Train epoch 7: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0325, accuracy: 9908/10000 (99%)\n",
      "Train epoch 8: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0322, accuracy: 9906/10000 (99%)\n",
      "Train epoch 9: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0330, accuracy: 9910/10000 (99%)\n",
      "Train epoch 10: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0302, accuracy: 9906/10000 (99%)\n"
     ]
    }
   ],
   "source": [
    "# 训练教师网络\n",
    "\n",
    "teacher_model, teacher_history = teacher_main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 小插曲，看看老师的暗知识 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def softmax_t(x, t):\n",
    "    x_exp = np.exp(x / t)\n",
    "    return x_exp / np.sum(x_exp)\n",
    "\n",
    "test_loader_bs1 = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data/MNIST', train=False, download=True, transform=transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])),\n",
    "    batch_size=1, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output (NO softmax): [-31.14481   -30.600847   -3.2787514 -20.624037  -31.863455  -37.684086\n",
      " -35.177486  -22.72263   -16.028662  -26.460657 ]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAV2ElEQVR4nO3de5TXdZ3H8efLQbybIuhRIIdWvJBHDzah5VaeNRPThTxp4m7e1mLbRLHsFOpZt7W2Q1mZbuZGalnregndmpLCVOxyNpEBtURAJ1MZJQWvlCkg7/3j+52ZH3NhvjP8fvOd+fxej3M887185vt785V58ZnP9/JRRGBmZsPfdmUXYGZm1eFANzNLhAPdzCwRDnQzs0Q40M3MEuFANzNLhAPdBoWkqZJWSWqVNKfsesxSJN+HbrUmqQF4DDgOaAOWAKdHxKOlFmaWmBFlF2B1YQrQGhFPAEi6BZgO9BjoI7VD7Mgug1je0LKel9ZFxJiy67Dhx4Fug2EssLpivQ04srfGO7ILR+rYmhc1VN0d858quwYbnhzoNhjUw7YtxvokzQRmAuzIzoNRk1lyfFHUBkMbML5ifRzwbGWDiJgXEU0R0bQ9OwxqcWapcKDbYFgCTJQ0QdJIYAbQXHJNZsnxkIvVXERskjQLWAg0ADdExPKSyzJLjgPdBkVELAAWlF2HWco85GJmlggHuplZIhzoZmaJcKCbmSXCF0WtLjWMyZ6sf+7kA7rt23nt5uzr/y4e1JrMtpV76GZmiXCgm5klwkMulq7tGjoWN7/nMAD+POdVAL59yE0AHLR9Q7dvW795AwCzPvP3Hdvavj4RgN3ufDg73uuv16Bgs23jHrqZWSLcQ7fkbH7fZAD+eNKOHduW/8PVW7TZju0B+OK6wzq2Pf7nvQH4xL6LALixcWFn+6t/AcCUfc4HYO9v/V+1yzbbZu6hm5klwj10S8aL57wLgNs+fwUA+43o/TW8R10+C4B9bu2cNOnNl18CYO6BHwZg1Sc7Jw1aeeo1APzPZ78KwCl7fAaAcV9yT92GDvfQzcwS4UA3M0uEIqLvVmYFSLoBOAl4PiIOzbeNAm4FGoEngY9ExEtbO87uGhUDmVN0wTPLANicz273wBudM9/9003nAdD4r7/t93EBHr86mwJ11Ye/BcC6N/8KwBlnXtDRpuG+ZQM6dld3x/ylEdFUlYNZXXEP3arpe8DULtvmAPdExETgnnzdzGrAF0WtaiLiV5Iau2yeDhyTL98I3Ad8rhaff9SDM7ZY3+vfOy+KNi4ZWM+83cGXrQTgi+/LbnO8ZPRDADw9tfMzJty3TR9hts3cQ7da2yci1gDkX/fuqZGkmZJaJLVs5I1BLdAsFe6h25AQEfOAeZCNoQ/kGKNOemzLY257WR02/yUbM//TG7tvsb3pPSs7ll+o4ueZDYR76FZrz0naFyD/+nzJ9Zglyz10q7Vm4Cxgbv71x+WWMzDrTz4CgKvH/me2nr/A65E7Dulosy9+yMjK5R66VY2km4HfAgdJapN0LlmQHyfpceC4fN3MasA9dKuaiDi9l139v6nczPrNgW6W2/iB7Fme55pGdtt308evzJeyX2o/+dQ0APb9modZbOjwkIuZWSLcQ7dkjdh/fMfyy0eOBeDATy8H4LrxvwQ6XxOQWdrrsbZX9gDRxngza/lANoPRAb5Z0YYQ99DNzBLhHrol57WTsxdpfeSLP+/YNnOPO7Zo89jGjQCc+cjZHdt2GZndirjw7T/sdsyNeUd+M5sBOP/47Nh3HfiOjjZvPvaHbazcbNu4h25mlggHuplZIjzkYslomHQgAOfNvQ2Ak3ftfMtA+5Od77xrNgCTLl0NwKg/db7/pfXKo7KFt3c/9tObsne5XLx6OgDfa/wZAMff1TmF3Qfvzd6NfvDsVQBsXr9+G/40Zv3nHrqZWSLcQ7dkrJyzK9DZM7/25Ykd++649AMAHPijBwCIPd4CwGPXd04MtPgDX8uXuj9YNGvaxwHY/PAKAN7zz1lP/6RP/qrz84+/FoAv/Dp778uST0zuPMD9vxvAn8isf9xDNzNLhHvolqxbnursfY9avg6Apy55NwCfOzMbZz9tt7srviPrmd/9190AuGL2GR17dnh4yRbHHv3tbAakJbeO7tg26fPnA7Dy1GsAuP3Gto59lz7wIQAOOOPBAf5pzPrmHrqZWSLcQ7dk7LUoezz/uWOyKex+ffitnTvv27LtdgiAzRV9msmLzwRgp59ksxKNurPveUjffPmVjuUDLrwfgGMX/QsAR1y2rGPfyr+7DoBDbz0HgLed05p9/muv9fkZZkW5h25VI2m8pEWSVkhaLml2vn2UpF9Iejz/umfZtZqlyIFu1bQJuCgiDgGOAs6TNAmYA9wTEROBe/J1M6syRVRzKl2zTpJ+DHwz/++YiFiTzyt6X0Qc1Nv37a5RcaQGPifG5vdltws+c/7Gjm1fOvxHAJyw80sA/O2D/wjApgWdFzX3uT5722K88caAP7vSiPHjOpYPbc4ukF6+d3Zx9bDfnAtA42ndb2e8O+YvjYimbjvM+uAeutWEpEZgMrAY2Cci1gDkX/fuof1MSS2SWjZSnUA1qzfuoVvVSdoV+CXwHxFxh6SXI2KPiv0vRUSv4+jb2kPvScPovbLP3mknADatbtta86obMXY/AP5wZVbHw0ffAMC0se/s1tY9dBso99CtqiRtD9wO3BQR7e+sfS4faiH/+nxv329mA+fbFq1qJAm4HlgREV+v2NUMnAXMzb/+eLBre3NduTMLbXrmWQD+5qKsD3X01OxFXnvR962RZkU50K2ajgbOAH4v6aF82yVkQX6bpHOBp4FTS6rPLGkOdKuaiPgN5E/sdFfdQXEz68aBbjaI2i/G7vWdwb0oa/XBF0XNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4Tfh25DjqS1wFPAaGBdyeX0VzVq3j8ixlSjGKsvDnQbsiS1DLeJHoZjzZYOD7mYmSXCgW5mlggHug1l88ouYACGY82WCI+hm5klwj10M7NEONDNzBLhQLchR9JUSasktUqaU3Y9vZE0XtIiSSskLZc0O98+StIvJD2ef92z7FqtPngM3YYUSQ3AY8BxQBuwBDg9Ih4ttbAeSNoX2DcilknaDVgKfAg4G3gxIubm/yDtGRGfK7FUqxPuodtQMwVojYgnImIDcAswveSaehQRayJiWb68HlgBjCWr98a82Y1kIW9Wcw50G2rGAqsr1tvybUOapEZgMrAY2Cci1kAW+sDe5VVm9cSBbkONetg2pMcFJe0K3A5cGBGvll2P1a8+x9Al3QCcBDwfEYf2sF/AVcAHgdeAs9t/Dd2a0aNHR2Nj40BqNjOrW0uXLl3X28vbRhT4/u8B3wS+38v+E4CJ+X9HAtfmX7eqsbGRlpaWAh9vZmbtJD3V274+h1wi4lfAi1tpMh34fmTuB/bIr/6bmdkgKtJD70tvF7HWdG0oaSYwE+Ctb31rFT66vjXOubNQuyfnnljjSsxsKKjGRdHCF7EiYl5ENEVE05gxfn+/mVk1VSPQ24DxFevjgGercFwzM+uHagR6M3CmMkcBr7Tfg2tmZoOnzzF0STcDxwCjJbUB/wZsDxAR/wUsILtlsZXstsVzalWsmZn1rs9Aj4jT+9gfwHlVq8jMzAbET4qamSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSWiUKBLmipplaRWSXN62H+2pLWSHsr/+1j1SzUzs60Z0VcDSQ3ANcBxQBuwRFJzRDzapemtETGrBjWamVkBRXroU4DWiHgiIjYAtwDTa1uWmZn1V5FAHwusrlhvy7d19WFJv5M0X9L4qlRnZmaFFQl09bAtuqz/BGiMiMOAu4EbezyQNFNSi6SWtWvX9q9SMzPbqiKB3gZU9rjHAc9WNoiIFyLijXz1O8A7ejpQRMyLiKaIaBozZsxA6jUzs14UCfQlwERJEySNBGYAzZUNJO1bsToNWFG9Es3MrIg+73KJiE2SZgELgQbghohYLulyoCUimoELJE0DNgEvAmfXsGYzM+tBn4EOEBELgAVdtl1WsXwxcHF1SzMzs/7wk6JmZolwoJuZJcKBbmaWCAe6mVkiHOhmZolwoJuZJcKBbmaWCAe6mVkiHOhmZolwoJuZJcKBbmaWCAe6mVkiHOhmZolwoJuZJcKBbmaWCAe6mVkiHOhmZolwoJuZJcKBbmaWCAe6mVkiHOhmZolwoJuZJaJQoEuaKmmVpFZJc3rYv4OkW/P9iyU1VrtQMzPbuj4DXVIDcA1wAjAJOF3SpC7NzgVeiogDgCuBL1e7UDMz27oiPfQpQGtEPBERG4BbgOld2kwHbsyX5wPHSlL1yjQzs76MKNBmLLC6Yr0NOLK3NhGxSdIrwF7AuspGkmYCM/PVP0taNZCiezG66+fVqW7nQfX3+5L/LmR8HtI8B/v3tqNIoPfU044BtCEi5gHzCnxmv0lqiYimWhx7OPF58Dlo5/NQf+egyJBLGzC+Yn0c8GxvbSSNAN4CvFiNAs3MrJgigb4EmChpgqSRwAyguUubZuCsfPkU4N6I6NZDNzOz2ulzyCUfE58FLAQagBsiYrmky4GWiGgGrgd+IKmVrGc+o5ZF96ImQznDkM+Dz0E7n4c6OwdyR9rMLA1+UtTMLBEOdDOzRCQR6H29miB1ksZLWiRphaTlkmaXXVNZJDVIelDST8uupSyS9pA0X9LK/O/Eu8quqQySPpX/PDwi6WZJO5ZdU60N+0Av+GqC1G0CLoqIQ4CjgPPq8By0mw2sKLuIkl0F/DwiDgYOpw7Ph6SxwAVAU0QcSnZDRxk3awyqYR/oFHs1QdIiYk1ELMuX15P9AI8tt6rBJ2kccCJwXdm1lEXS7sB7ye48IyI2RMTL5VZVmhHATvmzMTvT/fmZ5KQQ6D29mqDuwqxd/qbLycDicispxTeAzwKbyy6kRG8D1gLfzYeerpO0S9lFDbaIeAb4KvA0sAZ4JSLuKreq2ksh0Au9dqAeSNoVuB24MCJeLbuewSTpJOD5iFhadi0lGwEcAVwbEZOBvwD1eF1pT7Lf1CcA+wG7SPpouVXVXmn3oY8ePToaGxtL+Wwzs+Fq6dKl6yJiTE/7irycqyYaGxtpaWkp6+PNzIYlSU/1ti+FIRczM6PEHrptu8Y5dxZq9+TcE2tciZkNBe6hm5klwoFuZpYIB7qZWSIc6GZmiXCgm5klwoFuZpYIB7qZWSIc6GZmiXCgm5klolCgF50RSNIpkkJSU/VKNDOzIvoM9KIzAknajWyGkHp8D7eZWemK9NCLzgj0BeArwOtVrM/MzAoqEuh9zggkaTIwPiK2OjGvpJmSWiS1rF27tt/FmplZ74oE+lZnBJK0HXAlcFFfB4qIeRHRFBFNY8b0+H52MzMboCKB3gaMr1gfx5aTre4GHArcJ+lJslnnm31h1MxscBUJ9CXAREkTJI0EZgDN7Tsj4pWIGB0RjRHRCNwPTIsIT0dkZjaI+gz0iNgEzAIWAiuA2yJiuaTLJU2rdYFmZlZMoRmLImIBsKDLtst6aXvMtpdlZmb95SdFzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBGFAl3SVEmrJLVKmtPD/k9LelTS7yTdI2n/6pdqZmZb02egS2oArgFOACYBp0ua1KXZg0BTRBwGzAe+Uu1Czcxs64r00KcArRHxRERsAG4Bplc2iIhFEfFavno/2UTSZmY2iIoE+lhgdcV6W76tN+cCP+tph6SZkloktaxdu7Z4lWZm1qciga4etkWPDaWPAk3AFT3tj4h5EdEUEU1jxowpXqWZmfWpyCTRbcD4ivVxwLNdG0l6P3Ap8L6IeKM65ZmZWVFFeuhLgImSJkgaCcwAmisbSJoMfBuYFhHPV79MMzPrS5+BHhGbgFnAQmAFcFtELJd0uaRpebMrgF2BH0p6SFJzL4czM7MaKTLkQkQsABZ02XZZxfL7q1yXmZn1k58UNTNLRKEeupnZUNc4585C7Z6ce2KNKymPe+hmZolwoJuZJcKBbmaWCAe6mVkiHOhmZonwXS42YL6rwGxocQ/dzCwR7qGb2Tbzb2tDg3voZmaJcKCbmSXCQy5mZlVS9tCTe+hmZoko1EOXNBW4CmgArouIuV327wB8H3gH8AJwWkQ8Wd1SO5X9r+BQqcH8/8GsUp+BLqkBuAY4jmw6uiWSmiPi0Ypm5wIvRcQBkmYAXwZOq0XBZtZdkX/Y/I9a+or00KcArRHxBICkW4DpQGWgTwc+ny/PB74pSRHR42TSZinxbwk2VBQZQx8LrK5Yb8u39dgmn7LuFWCvahRoZmbFqK9OtKRTgeMj4mP5+hnAlIg4v6LN8rxNW77+h7zNC12ONROYma8eBKyq1h8EGA2sq+LxhiufB5+Ddj4PaZ6D/SNiTE87igy5tAHjK9bHAc/20qZN0gjgLcCLXQ8UEfOAeUUq7i9JLRHRVItjDyc+Dz4H7Xwe6u8cFBlyWQJMlDRB0khgBtDcpU0zcFa+fApwr8fPzcwGV5899IjYJGkWsJDstsUbImK5pMuBlohoBq4HfiCplaxnPqOWRZuZWXeF7kOPiAXAgi7bLqtYfh04tbql9VtNhnKGIZ8Hn4N2Pg91dg76vChqZmbDgx/9NzNLRBKBLmmqpFWSWiXNKbuewSZpvKRFklZIWi5pdtk1lUVSg6QHJf207FrKImkPSfMlrcz/Tryr7JrKIOlT+c/DI5JulrRj2TXV2rAP9IpXE5wATAJOlzSp3KoG3Sbgoog4BDgKOK8Oz0G72cCKsoso2VXAzyPiYOBw6vB8SBoLXAA0RcShZDd0JH+zxrAPdCpeTRARG4D2VxPUjYhYExHL8uX1ZD/AXZ/mTZ6kccCJwHVl11IWSbsD7yW784yI2BARL5dbVWlGADvlz8bsTPfnZ5KTQqAXeTVB3ZDUCEwGFpdbSSm+AXwW2Fx2ISV6G7AW+G4+9HSdpF3KLmqwRcQzwFeBp4E1wCsRcVe5VdVeCoGuHrbV5a07knYFbgcujIhXy65nMEk6CXg+IpaWXUvJRgBHANdGxGTgL0A9Xlfak+w39QnAfsAukj5ablW1l0KgF3k1QfIkbU8W5jdFxB1l11OCo4Fpkp4kG3b7O0n/XW5JpWgD2iKi/Te0+WQBX2/eD/wxItZGxEbgDuDdJddUcykEepFXEyRNksjGTFdExNfLrqcMEXFxRIyLiEayvwP3RkTyPbKuIuJPwGpJB+WbjmXLV13Xi6eBoyTtnP98HEsdXBwe9nOK9vZqgpLLGmxHA2cAv5f0UL7tkvwJX6s/5wM35R2cJ4BzSq5n0EXEYknzgWVkd4E9SB08NeonRc3MEpHCkIuZmeFANzNLhgPdzCwRDnQzs0Q40M3MEuFANzNLhAPdzCwRDnQzs0T8PyNaDvQ8wWUUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "teacher_model.eval()\n",
    "with torch.no_grad():\n",
    "    data, target = next(iter(test_loader_bs1))\n",
    "    data, target = data.to('cuda'), target.to('cuda')\n",
    "    output = teacher_model(data)\n",
    "\n",
    "test_x = data.cpu().numpy()\n",
    "y_out = output.cpu().numpy()\n",
    "y_out = y_out[0, ::]\n",
    "print('Output (NO softmax):', y_out)\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(3, 1, 1)\n",
    "plt.imshow(test_x[0, 0, ::])\n",
    "\n",
    "plt.subplot(3, 1, 2)\n",
    "plt.bar(list(range(10)), softmax_t(y_out, 1), width=0.3)\n",
    "\n",
    "plt.subplot(3, 1, 3)\n",
    "plt.bar(list(range(10)), softmax_t(y_out, 10), width=0.3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 让老师教学生网络 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StudentNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(StudentNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(28 * 28, 128)\n",
    "        self.fc2 = nn.Linear(128, 64)\n",
    "        self.fc3 = nn.Linear(64, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.flatten(x, 1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        output = F.relu(self.fc3(x))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 关键，定义kd的loss "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distillation(y, labels, teacher_scores, temp, alpha):\n",
    "    return nn.KLDivLoss()(F.log_softmax(y / temp, dim=1), F.softmax(teacher_scores / temp, dim=1)) * (\n",
    "            temp * temp * 2.0 * alpha) + F.cross_entropy(y, labels) * (1. - alpha)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_student_kd(model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    trained_samples = 0\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        teacher_output = teacher_model(data)\n",
    "        teacher_output = teacher_output.detach()  # 切断老师网络的反向传播，感谢B站“淡淡的落”的提醒\n",
    "        loss = distillation(output, target, teacher_output, temp=5.0, alpha=0.7)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        trained_samples += len(data)\n",
    "        progress = math.ceil(batch_idx / len(train_loader) * 50)\n",
    "        print(\"\\rTrain epoch %d: %d/%d, [%-51s] %d%%\" %\n",
    "              (epoch, trained_samples, len(train_loader.dataset),\n",
    "               '-' * progress + '>', progress * 2), end='')\n",
    "\n",
    "\n",
    "def test_student_kd(model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.cross_entropy(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest: average loss: {:.4f}, accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "    return test_loss, correct / len(test_loader.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def student_kd_main():\n",
    "    epochs = 10\n",
    "    batch_size = 64\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=False, download=True, transform=transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,))\n",
    "        ])),\n",
    "        batch_size=1000, shuffle=True)\n",
    "\n",
    "    model = StudentNet().to(device)\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "    \n",
    "    student_history = []\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        train_student_kd(model, device, train_loader, optimizer, epoch)\n",
    "        loss, acc = test_student_kd(model, device, test_loader)\n",
    "        student_history.append((loss, acc))\n",
    "\n",
    "    torch.save(model.state_dict(), \"student_kd.pt\")\n",
    "    return model, student_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 1: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1446, accuracy: 9705/10000 (97%)\n",
      "Train epoch 2: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1195, accuracy: 9739/10000 (97%)\n",
      "Train epoch 3: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0880, accuracy: 9799/10000 (98%)\n",
      "Train epoch 4: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0725, accuracy: 9825/10000 (98%)\n",
      "Train epoch 5: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0753, accuracy: 9831/10000 (98%)\n",
      "Train epoch 6: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0711, accuracy: 9846/10000 (98%)\n",
      "Train epoch 7: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0691, accuracy: 9837/10000 (98%)\n",
      "Train epoch 8: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0706, accuracy: 9832/10000 (98%)\n",
      "Train epoch 9: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0658, accuracy: 9841/10000 (98%)\n",
      "Train epoch 10: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0664, accuracy: 9842/10000 (98%)\n"
     ]
    }
   ],
   "source": [
    "student_kd_model, student_kd_history = student_kd_main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 让学生自己学，不使用KD\n",
    "def train_student(model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    trained_samples = 0\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        trained_samples += len(data)\n",
    "        progress = math.ceil(batch_idx / len(train_loader) * 50)\n",
    "        print(\"\\rTrain epoch %d: %d/%d, [%-51s] %d%%\" %\n",
    "              (epoch, trained_samples, len(train_loader.dataset),\n",
    "               '-' * progress + '>', progress * 2), end='')\n",
    "\n",
    "\n",
    "def test_student(model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.cross_entropy(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest: average loss: {:.4f}, accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "    return test_loss, correct / len(test_loader.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def student_main():\n",
    "    epochs = 10\n",
    "    batch_size = 64\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=False, download=True, transform=transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,))\n",
    "        ])),\n",
    "        batch_size=1000, shuffle=True)\n",
    "\n",
    "    model = StudentNet().to(device)\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "    \n",
    "    student_history = []\n",
    "    \n",
    "    for epoch in range(1, epochs + 1):\n",
    "        train_student(model, device, train_loader, optimizer, epoch)\n",
    "        loss, acc = test_student(model, device, test_loader)\n",
    "        student_history.append((loss, acc))\n",
    "\n",
    "    torch.save(model.state_dict(), \"student.pt\")\n",
    "    return model, student_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 1: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1245, accuracy: 9627/10000 (96%)\n",
      "Train epoch 2: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0980, accuracy: 9717/10000 (97%)\n",
      "Train epoch 3: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1002, accuracy: 9726/10000 (97%)\n",
      "Train epoch 4: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0838, accuracy: 9774/10000 (98%)\n",
      "Train epoch 5: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1062, accuracy: 9760/10000 (98%)\n",
      "Train epoch 6: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0989, accuracy: 9789/10000 (98%)\n",
      "Train epoch 7: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1106, accuracy: 9756/10000 (98%)\n",
      "Train epoch 8: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1024, accuracy: 9794/10000 (98%)\n",
      "Train epoch 9: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1087, accuracy: 9794/10000 (98%)\n",
      "Train epoch 10: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1110, accuracy: 9797/10000 (98%)\n"
     ]
    }
   ],
   "source": [
    "student_simple_model, student_simple_history = student_main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x159ccce2cf8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3xUVfr/389MJr2QQiChBekQIFKUJgIKwsIKgmsBRLeIumtf/CnuWlcUu7KifF11UUFRsStNIJSlKARBWjRAAoQAAUJ6m3J+f9zJZBISksDApJz36zWvufeec8957s3kc859zrnPEaUUGo1Go2m8mLxtgEaj0WguLFroNRqNppGjhV6j0WgaOVroNRqNppGjhV6j0WgaOVroNRqNppGjhV6j0WgaOVroNfUCEcl3+zhEpMhtf8p5lLtZRKZ60laNpqHh420DNBoApVRw2baIpAF/UUqt9J5FFwcR8VFK2bxth6Zxo3v0mgaBiJhF5DEROSAiJ0VkoYg0c6YFicgiEckSkWwR+VFEwkXkZaA/8I7zyeDlKsr1EZHPReS489xEEenilh4kInNE5LCI5IjIWhHxcaYNcz4x5IjIIRGZ7Dxe4SlCRO4UkZXObX8RUSJyl4jsB3Y5j78lIukikisiP4nIgEo2PuG89lwR2SIiLUXkXRGZVel6fhCROz146zWNAC30mobCQ8AoYAjQGrACrzrT/oLxdNoKiALuBkqVUn8HtmA8HQQ796viG6AD0BJIBt53S5sDdMVoMCKAfwJKRDoC3wEvApFAX2B3Ha5nnPOcS537m4CezrK+Bj4TEYszbSYwwXn9zYDpQLHTzskiIgAiEgsMBj6tgx2aJoB23WgaCncAU5VSGQAi8hSwW0T+hCH6zYEOSqldGOJeK5xuE5ewO8vNEBF/wA5MA+KVUsecWdY7890CfKuU+tx5/ITzU1tmKaWy3ez4wM2GZ4F/AJcAv2I0ZNOVUvucWX525lsPKIzGbz0wGVimlMqqgx2aJoDu0WvqPc4eaxtgidO9ko0hdiaMHvC7wFpgsdP98ayImGtZto+IvFTmFsHo0Yuz3BiMztCBKk5tA+w/j8s6XMmOmSLyq4jkAKcBfyDKee2tqqpLGREJPwDK3ERTgQ/PwyZNI0ULvabe4xS0I8AIpVQzt4+/UuqkUqpEKfW4UqorMBT4A3BT2ek1FP9HDJfIcCAMw00DhtgfBWwYPevKHMZw91RFARDott+yqssq2xCRkcA9wHUYrpkIoAgQt2uvrq4PgOtFpC9G4/N9Nfk0TRgt9JqGwjxgtoi0ARCRaBH5vXP7ahHpLiImIBdDnO3O845TtVCXEYLh7z4FBAHPlCUopawYQvq6iLRwDggPcT4tfACME5HrnMebi0gv56nbMcTXX0S6ArfVcG0hGO6nE4Av8DRGj76Md4BnReQSMbi0bCBaKXUA2AP8F/hEKVVaQ12aJogWek1D4QVgJbBaRPKAjUAfZ1orjAHMPIxZLEsoH5B8FZgmIqdF5IUqyn0XQ2CPATuB/1VKvxfDbfIzRmPwL4ye9n5gPPAohqtlK9DDzVYfZ7lvAwtquLZvgXXOeg4AJ6no75+N0VNfjdGQzQP83NLfxxjI1W4bTZWIXnhEo2nYiMgo4E2lVEdv26Kpn+gevUbTgBERX4ynjre9bYum/qKFXqNpoIhIAobbKASY62VzNPUY7brRaDSaRo7u0Ws0Gk0jp969GRsVFaXi4uK8bYZGo9E0KJKSkk4qpZpXlVbvhD4uLo6tW7d62wyNRqNpUIjIwerStOtGo9FoGjla6DUajaaRU+9cNxqNpnFSbLXz2/E8ko/msedoLnuP5pJ+uogQfx+aBVpoFuBLeJCFZoG+hDv3mwVaCA8y9sOc+xaz7p/WFS30Go3GoyilyMwrcYn53qN57D2aS+rJAuwOYzp3gMVMl5Yh9I8Lp6DUTnZhKftP5HP6oJXswlJsjuqnfYf4+dAsyEJ4oC9hAcZ3eKDRQDQLNPbdv5sF+hLq74MzbH+TRAu9RqM5Z0ptDlIy81xinnzMEPasgvLYaq2aBdAtJoQx8S3pFhNKt5hQ2kYEYjZVLbxKKfJLbGQXWskutHK6sJTThaXkFFk5XWDsZxeWkl1k5XShlUNZhZwuKCW3uPoVGc0moVmAhbDASg1DgPHEEOznQ31oB6KC/fhdzxiPl1sroReR0cDrgBl4Ryk1u1J6O+A9jMUfsjAWiEh3pj0PjHVm/ZdS6hMP2a7RNCgcDkX66SJEoFmgxSku9UBdasnJ/BJnD728l74vM9/V+/bzMdGlZQhXd4t2CXq3lqGEBVpqKLkiIkKIv4UQfwttImp/nt2hjMagrCEoNBqCbGdD4d5wHMkuZk9GLqcLrRRZ7TUXfpFIaNPMO0LvDMk6FxgJpANbROQbpdQet2wvAR8opd4XkRHAc8AtIjIWI8JgAka0vbUislQplevpC9Fo6hNWu4OU4/nszshhd0YuuzNy2Hs0j/yS8l6nj0lcroXKPUyXz7osPajcZ+1vqdWaKudl+4ETBYagHysX9RN5Ja48LUL96BYTyvCuhqh3jwkhLjIIHy/6z80mISLIl4gg3zqdV2y1U1BSP9Zn9zFdmPtXmx79ZcA+Z9xrRGQRRnhWd6HvDjzg3E4EvnI7vta5XJtNRHYAo9FrWmoaEYWlNvYezWOPU9R3ZeTw27F8Su0OAAJ9zXSPCWVSn1Z0jw3FJGL0LotKy3ucBVYOZxWyy9kjLbY6qq0vwGI2BiedDUS5L7psu6yBKGtEDF92Va6S7MJSpy89z9VbTzlebruv2UTH6GCGdmpOt5gQuseE0jUmtM5iWp/xt5gveOPpbWoj9K2ouOxZOnB5pTw7gEkY7p3rgBARiXQef0JEXsFYcWc4FRsIAERkOsaCx7Rt27aOl6DRXDyyC0tdPfTdGbnsOpJD6skCysYOwwMt9IgN449D4ugRG0aP2FDiIoOq9UdXR7HV7nI3VP42XBHl38nHcp0Nh9U12FkVof4+zqcFXwItZtJOFXA0p9iVHhXsR7eYEG4bHEe3mBC6xYTSoXmwnuXSCKiN0Ff1C638a5oBvCEit2EsoHAEsCmlVohIf4xFIk5grHR/xjOSUuptnGFW+/Xrp6OsabyOUorjuSXsOpJTQdiPZBe58sSG+dM9NoxxvWKJb2WIekyYv0f87v4WMzFhAcSEBdT6HIdDkVdiI8dtALOqBuJ0YSn5JTYubx9R7kuPCaV5iF/NlWgaJLUR+nSMtSjLaA1kuGdQSmUAEwFEJBiYpJTKcabNAmY50z4CUs7fbI3GczgcioNZhRVEfU9GLqecM0dEoH1UEH3ahXPLwHb0iA2lR2xYvXNfmExCWICFsAALbSMDaz5B02SojdBvATqJSHuMnvpNwGT3DCISBWQppRzATIwZOGUDuc2UUqec62n2AlZ40P5GgcOhKLU7KLE5KLU5KLU7v1379vK0yunO7arOtTkciAgmAbOIc1swm8Dk3K+w7cxrMhn5TGKkGfvGYJd7ecZ5xvHK265zRTCbBB+z8W0xm/AxCT4mEz5mwWIWzCbjmMVscuYRfFz5xFWvJ6hpkNRiFjpFh3BVt2iX66VbTChBfnomsqbhUuOvVyllE5G7geUY0yvfU0rtFpGnga1KqW+AYcBzIqIwXDd/c55uAdY7/0lzMaZd1o/h7QuAze5g5d5Mvt95lIISm0twS1ziazcE3VpZkD3nrbKYBV+zCV8fEz5mE0qBQynj41A4Kuy7bddzh5mPs7GwOBsIs8nkbBCcjYbJrXFw5itrYMoakKM5RWcMknaLCWVin1bEx4bRPTaUzi1C8PXRPmlN46LeLTzSr18/1dCiV2YXlvLJlsN8sOkgR7KLaB7iR4tQP5fg+vqY8TWb8PNx7ruOV9z383HL42PC12yuMk+1ZZhNmOo46FeGUsrVKNid23ZHeSPgcN9WVR+3OxTKbdvhLMfmcGB3KKx2hc1hNGw2u8Jmd2B1KOwOh5Fmd9t2ftsd7vkUVrvDONd53OY85l7+GfkcxnZUsB89YkPp7nS9tI+q+yCpRlNfEZEkpVS/qtL08+h58NvxPP67IY0vf06n2OpgwCURPDauO1d3i/bqfOJzQZyuFxOifxQaTSND/0/XEbtDsTo5k/kbU9mw7xR+PiYmJLTi1kFxdI8N9bZ5Go1GcwZa6GtJTpGVz7Ye5v1NaRzOKiImzJ//N7oLN/VvW+9mX2g0Go07WuhrYF9mPu9vTOPzbekUltrpHxfOI6O7MapHC/0iiUajaRBooa8Ch0Ox9rcTvLchlfUpJ/E1m/h971j+ODiO+FZh3jZPo9Fo6oQWejfyiq0sTkrn/Y1ppJ0qJDrEj7+P7MzNl7clKli/NajRaBomWuiB1JMFvL8xjcVJ6eSX2Li0bTMeGNmZMfExek61RqNp8DRZoXc4FOv3nWT+hlQSfz2BxSyM6xXLrYPiSGjTzNvmaTQajcdockJfUGLji23pzN+Yxv4TBUQF+3HfVZ2YMqAt0SH+3jZPo9FoPE6TEfpDpwp5f1Man249TF6xjV6tw3j1xt78rmcMfj6NOxa1RqNp2jRqoVdKsXH/Kf67IY1VyccxizCmZwy3DYqjT9tmDWoZN00DoyQPslLhdGrF7+yDYPKBoOYQGAlBURAY5fYd6fx2pvvodzQ050+jFPrCUhtf/nyE9zem8dvxfCKDfLl7eEemXN6OlmHaPaPxAEpBwUnIOnCmmJ9OhYITFfMHRkJ4e2jVDxw2KDwFp/bD4R+NbVXNilJ+YW7iH1XeOAQ1r9QwOBsKi/5914hSUJpv/P0KTxl/q5I8cNiNv4NygHLbdjjOPO7KqyrldS/DUc1xt3MrH4+4BK563OOX3KiEPv10IR9uOsiiLYfJKbLSPSaUF6/vxe97xzb6pcI0FwCHHXLSnQJ+wE3M04zv0ny3zAJhrSE8DrqMMUQ9on35t/9Z3r9wOKDoNBSedIrPSTcRcts/fRCOJBnHHdUEgfUNrrkxKNv3CwHfIDD7GkH3GypKQXHOmfer8CQUOIW88j21l9Rcbl0Rk9vHbHybzMa9dT/mOm5yprkdr3Kdp/On0Qh92skCRry8BhFhdI+W3DY4jn7twrV7RnN2rEWGgJb1xt176NmHwGEtz2v2hWbtjF5X3OCKYh7eDnzO8V0Lk8kQ36BIaN6l5vxKQXG2IWJVNg4njO3cI3D0FyPNXnqW+n0MwfcNBktg+bZv2XZVaUHO9ODyPJYgt/xBYLac2/1wOIzrKxNll1BXvl63ffe/kzuWoPKGLSQGWvas2mXmH2b8HaoU40qfCsfdBbv+ak2jEfq4qCAeH9edkT1a0qpZ7Zdf0zRS7DYozTMeyUvyoDgX8jIquliyUo1j7viFGr3ylvHQ7feGqJeJeWis8U/ubUQgINz40LHm/EoZ96CyOJYWGE8lpQVgLSzfLi00vvMzK6aV5Bvuhdpi9q2ikajUiJgsUJRV6QnmVPX1+IWWC3VYa4jtXS7YVT3BWLQWgI5Hr6lv2Eqd4pxbLtKuT3XHqjhuLay+juAWZ7pWIi4xtgMj6nXPzKsoZTwZlBZU/FgLzjxW1ohYCys2KKXuDUqBUV5AeMUedoXedqXe97k+NTUBdDx6jXcoyYf0LXBqXzUiXYV424prLldMRs/OL9TwM/uFGEIQ3r583z2t7BPcwuit+wVf8EtvlIgYQuvjZzSImgaDFnqN5yg4CYc2wcFNcGij4R92fwQ3+RgC7B9aLsYhMRDVuZIoVyHS7scsgbrXrdHUAS30mnNDKWOw8tAmOLjR+D75m5Hm429MI7ziQWg7AFr0NAa7fPy0QGs0XkALvaZ2OBxwItnoqR/cZAh77hEjzT8M2gyAhMnQdhDEJmhfqkZTj9BCr6kaWykc3VEu7Ic3G3O9wXC3tB0I7QYZ39HdjalpGo2mXqKFXmNQNnBa5opJ3wq2IiMtsiN0HVcu7OFx2gWj0TQgaiX0IjIaeB0wA+8opWZXSm8HvAc0B7KAqUqpdGfaC8BYwAT8ANyn6tuczqZI2cDpoc2GsB/dYQycisl4qaTvbdBuoCHswdHetlaj0ZwHNQq9iJiBucBIIB3YIiLfKKX2uGV7CfhAKfW+iIwAngNuEZFBwGCglzPf/4ArgTWeuwRNjZxt4NTsB637wZAHDGFvfZkxK0aj0TQaatOjvwzYp5Q6ACAii4DxgLvQdwcecG4nAl85txXgD/hiBHGwAMfP32xNjRRmwe4vy4W9bODULwzaXg69bzZcMbGX6oFTjaaRUxuhbwUcdttPBy6vlGcHMAnDvXMdECIikUqpTSKSCBzFEPo3lFJ7K1cgItOB6QBt27at80Vo3LAWw0//B+tehpIcCG7pdMEMMr6ju9eP1/g1mgaCUgqbsmG1Wym1l1LqKK34bS/ft9qtFY5bHdYz8lY+5n5Ou9B2PHzZwx6/htoIfVWjbpV97DOAN0TkNmAdcASwiUhHoBvQ2pnvBxEZqpRaV6Ewpd4G3gYjBELtzde4cDhg52ew+l+Qcxg6jYIRjxn+dj1wqqln/Hb6N5amLuWnYz/hcFQTovkiYlf2swqwOkPyzg2TmPA1+WIxW/A1+eJrNj4WkwVfsy/NA5t7pJ7K1Ebo04E2bvutgQqRoJRSGcBEABEJBiYppXKcPfXNSql8Z9pSYABGY6DxFAfWwg+PGQOqLXvB+LlwyZXetkpTCbvDzoGcA0QHRhPmd5awxY2Uw3mHWZq6lKWpS9mXvQ+zmOndvDeB/oHeNq1aAT6bKNd0zH2/LN3H5J2JjrWpdQvQSUTaY/TUbwImu2cQkSggSynlAGZizMABOATcLiLPYTwZXAm85iHbNcf3wA+Pw74fIKwNXPc29PyDntNeT1BKkZ6Xzqajm9h8dDM/HfuJnJIczGLm0uhLGdZmGMPaDKNdaDtvm3rBOFF4gmVpy1iaupSdJ3cC0Ce6D/+4/B+MbDeSyIBIL1vYNKhV9EoR+R2GQJuB95RSs0TkaWCrUuobEbkeY6aNwuit/00pVeKcsfMmMNSZtkwp9eDZ6tLRK2tB7lFInAXbF4JvCAz9O1x2h15dqB6QVZzFT0d/YvPRzWw+upkj+cYgeIvAFgyMHUjfFn05lHuINelrSDmdAkBcaBzD2wznyjZX0rt5b6/1+jxFTkkOKw+udLlmFIquEV0Z034MY+LGEBMc420TGyVni16pwxQ3JEryYMMc2PQG2K1w2e0w9CEdSdCLFNmK2HZ8m0vYk7OSAQixhHBZzGUMiBnAgJgBtAttd8YiOEfyj7Dm8BrWHl7LluNbsDlshPmFMbTVUK5scyWDYwcT7GtE2rRaraSnp1NcXIvonl7AoRyU2EsoshVRYitBofAx+RDgE0CAT0CDb7zqE/7+/rRu3RqLpeLCLlroGzp2K2x7H9bMNlbb6THRWFcyor23LWty2Bw29pza4xL27ZnbsTqsWEwWLo2+1CXs3SK71Unc8kvz2ZCxgbWH17LuyDpySnLwMfnQv0V/hrUZxqWWS2nerDmRkZH1ZtU0h3KQX5pPbmkueaV5OJQDH5MPYX5hhPmF4W/2rze2NhaUUpw6dYq8vDzat6/4/6+FvqGiFCR/DyufhFMpxhTJUc9A677etqxWFNuKWZq6lP3Z+4kJjqF1cGtaBbciNjiWQIv3B+Bqg1KKtNw0Q9gzNrPl2BbyrHkAdI3o6hL2Pi36EODjmdWMbA4bO07sYO3htSQeTiQtN41Xu79Ku47tCPUNJcQ3hACfAK+IqFKKAmsBuaW55JbkYld2zCYzob6hhPmFEegTqMX9AqOUIjk5mW7dulU4rhceaYikb4UVjxlBxaI6w00fG4tON4B/ohOFJ1j06yI++/UzTpecxsfkg63SYtYR/hG0Cm5V/gkxvlsHtyYmKAbLua436gFOFp3kx6M/unrtxwqOARAbFMuouFEMiBnAZTGXEeF/YVxmPiYf+rboS98WfXmw34Ok5aRx4uAJfEw+nCw6ycmik/iYfAj2DSbEEkKQJQjzBXw3QilFka2InNIccktysTlsmMREiG8IYX5hBFmCMImeAHCxOJeGVAt9fSPrAKx62nirNag5jH0F+twK5vr/p9p7ai8f7vmQpWlLsTvsXNnmSqZ1n0bfFn3JKs7iSP4RjuQdMb7zj5Cen87uU7tZeXAlNlXeEAhCdGC0IfwhrSs2CMGtiA6M9qiwFVoL2Xp8q0vYywZJQ31DuTzmcm7veTsDYwbSOqS1V3qrcWFxFFmKaB/WHpvDRr41n7zSPHJLcskuzkZECLIEEeIbQoglxGONZLGtmJySHHJKc7DarYgIwZZgwvzCCPEN0eLegKj/6tFUKMyCtS/AlnfAbIErH4ZB9xgrKtVj7A47a9LX8OGeD0k6nkSATwA3dL6BKd2m0Da0/C3nqIAoogKi6N28d5VlZBZmkp6fzpH8I2TkZxgNQV46Px79kczCzAovrPiYfIgJinEJf1ljEBscS6vgVkT6n92PbXVY2X1ytzHtMWMzv5z4BZuy4WvypU+LPoztM5YBsQPoGt71gvaUzwUfkw/N/JrRzK8ZDuWg0FpInjWPvNI88vPzOcpR/H38DdH3Damzn7zUXuoS9xJbCQBBliB8S3z57vPvuPtvd3vsWp588kmCg4OZMWOGx8rUVI0Wem9jLYIf/w/WvwKleXDpVBj2KITW7yloBdYCvkz5koV7F5Ken05MUAwz+s3guk7XEepbt6BoZpOZmOAYYoJj6E//M9JL7aUcLTjqehJwfypIPJxIVnFWhfwBPgHEBsW63EFln+OFxw0/+/EtFFgLEITukd25tcetDIgdQELzBPx9Gs4UVZOYCPYNJtg3mJaBLSmxl5BXmkeeNY8ThSc4UWi4e8pEvzoXi9VhJbckl5zSHIqsRmjqAEsALYNaEuoXisVkIe10GvPemudRoT9f7HY7ZnP9aojrK1rovYXDATs/hVX/gtx06HQNjHwKorvVfK4XOZJ/hIV7F/JlypfkW/NJaJ7AA30fYETbERdsCp2v2Zd2oe2qfbGo0FpY/hRQ6ang5+M/uwZPAdqGtGVse6PHflnLyxrNG6oigr+PP/4+/jSnOTaHzRD90jxySnI4XXwak5hcLp4gSxAF1gJySnIosBYA4Ofj53pr19fsW6H8Rx55hP3795OQkMDIkSOJjo7m008/paSkhOuuu46nnnoKgAkTJnD48GGKi4u57777mD59OgDLli3j0UcfxW63ExUVxapVqwDYs2cPw4YN49ChQ9x///3ce++9ACxYsIA5c+ZQWlrK5ZdfzptvvonZbCY4OJgHH3yQ5cuX8/LLLzNkyJCLdYsbNFrovcH+RCNkwbGdENMbrnsL2g/1tlXVopTi58yfWbB3AasOrcKEiZFxI7ml2y30bN7T2+YRaAmkY3hHOoZ3rDI9pySHjPwMQv1CaRXc6iJb53me+nY3ezJy63SOXdmxO+zYlQ33mXYiJnxMZuJjw3l6fNX3D2D27Nns2rWL7du3s2LFChYvXsxPP/2EUoprr72WdevWMXToUN577z0iIiIoKiqif//+TJo0CYfDwe233866deto3749WVnlT2DJyckkJiaSl5dHly5duOuuu9i3bx+ffPIJGzZswGKx8Ne//pWFCxcybdo0CgoKiI+P5+mnn677jWvCaKG/mBzf7QxZsBLC2sLEdyB+Ur0NWWC1W1l+cDkL9ixg96ndhPqG8scef+SmrjfRMqilt82rNWXzupsyZjE73Ry+OJQDh7JjErPLlVOXsYgVK1awYsUKLr30UgDy8/NJSUlh6NChzJkzhy+//BKAw4cPk5KSwokTJxg6dKhr3ndERPlspbFjx+Ln54efnx/R0dEcP36cVatWkZSURP/+hhuvqKiI6Ghj8Ruz2cykSZPO+340NbTQXwxyM5whCz4yBldH/gsum15vQxZkF2fz2W+fsSh5EZlFmcSFxvHYgMcYd8m4BjP/vTHzxO97eLV+pRQzZ87kjjvuqHB8zZo1rFy5kk2bNhEYGMiwYcMoLi5GKVXtgLCfX/laCGazGZvNeOK49dZbee65587I7+/vr/3y50D97Eo2FopzDR/8nD7wy6cw4K9w73YYfG+9FPkD2Qd4atNTjFw8kjk/z6FDsw7MvWouX0/4mhu63KBFvgkTEhJCXp4x1nHNNdfw3nvvkZ+fD8CRI0fIzMwkJyeH8PBwAgMDSU5OZvPmzQAMHDiQtWvXkpqaClDBdVMVV111FYsXLyYzM9OV/+DBgxfq0poEukd/IbBbIWm+EbKg8KThnrnqcWNR7XqGUoqNGRv5cO+HbDiyAV+TL7/v8HumdJtCp/BO3jZPU0+IjIxk8ODBxMfHM2bMGCZPnszAgQMBCA4OZsGCBYwePZp58+bRq1cvunTpwoABAwBo3rw5b7/9NhMnTsThcBAdHc0PP/xQbV3du3fnmWeeYdSoUTgcDiwWC3PnzqVdu8Yb5fNCo0MgeAql4EQyJH8H2z+GrP3QbgiMehpa1b+QBcW2Yr498C0L9yxkf85+ogKiuKnLTfyhyx8u2BufmnNn7969Z7zyrmm6VPV70CEQLhQOOxz+yRD3X5cYb7UCtO4P18yCzqPrXciCzMJMFiUv4rPfPiO7JJtuEd14dsizXBN3zRlT6jQaTeNAC31dsRYZKzolfwe/LjVcMyaLMT1y4N3Q5Xf18mWn3ad28+GeD1meuhy7sjO8zXCmdp9Kvxb9dBAqjaaRo4W+NhRmQcoKQ9z3rQJrIfiFQqeR0HUsdLwa/Ovf9D27w07i4UQ+3PMh2zK3EegTyI1db2RK1ym0CW1TcwEajaZRoIW+OrIPQfISQ9wPbgRlh5AY6H2zIe5xV4CP91wddoednFLjjces4iyyirM4XXzatX+65DS7Tu7iSP4RYoNimdFvBhM7TSTEt37HztFoNJ5HC30ZSsHxXUb89+Tv4dgvxvHmXWHI/dBlLMReesFebrI5bGSXZJ8h2K79ktMV0rJLsqtdmT7UN5QI/wjiQuP4e7+/M7zNcL3Cj0bThGna//12GxzaZAj7r98bvXgE2lxuvNTUdSxEdjinoq12q6tnXVVv27Xt/M4trfqVdkEI8wsj3D+cCLhPnCgAACAASURBVP8IOjTrQLhfOOH+4a5j4f7hhPsZ2838m2ExeS+Wu0ajqX80PaEvLYD9qw23zG9Loeg0mP2gw3C4YoaxuEdw9DkXv+TAEp7f8vwZERXLMImJZn7NXALdObwz4f7hRPpHVhRvp5g382tW70LlajRlvPbaa0yfPp3AwLq9TBccHOx64aquzJ8/n1GjRhEbG1vnc+fNm0dgYCDTpk07o5y4uDi2bt1KVFRUteevWbOGl156ie+++w6Af/7zn2zZsoVvvvmGa665hqNHj+Ln50dpaSlXX301zzzzDM2aNTun6/QkTUPoC07Cb8uMnvv+1WArNgZPO482eu0drgK/4POqwqEcvPHzG/xn53/o1bwXN3e92SXmrm+/CEL9QvWCDZpGw2uvvcbUqVPrLPTnw/z584mPjz8nob/zzjs9Ug7ArFmz2LBhA0uWLHGFcli4cCH9+vWjtLSUmTNnMn78eNauXXtO5XuSxiv0WQecg6nfw+HNoBwQ2tpYranrWGg3yFjgwwMUWAuYuX4miYcTmdRpEv+4/B9eXQpPo/E0BQUF3HDDDaSnp2O323nsscc4fvw4GRkZDB8+nKioKBITEyv01BcvXsx3333H/PnzSU1NZfLkydhsNkaPHl2h7BdffPGMkMdpaWmMGTOGIUOGsHHjRlq1asXXX3/N999/z9atW5kyZQoBAQFs2rSJgABjrd7MzEzGjBlDUlISO3bsICEhgYMHD9K2bVs6dOjAzp07eeGFFwgODnb13t3LAfj3v//Nt99+i9Vq5bPPPqNr165V3o+XX36ZJUuWsHz5clf97vj6+vLCCy/QsWNHduzYQe/eZy64czGpldCLyGjgdcAMvKOUml0pvR3wHtAcyAKmKqXSRWQ48Kpb1q7ATUqprzxhfAWUgqPbywdTM/cYx1vEw9CHDHFv2cvjLzCl56Vzz+p7SM1J5ZHLHmFy18l6XrrmwrL0ESPEtSdp2RPGzK42edmyZcTGxvL9998DkJOTQ1hYGK+88gqJiYlndXcA3Hfffdx1111MmzaNuXPnuo6vWLGClJSUM0Iet23blpSUFD7++GP+85//cMMNN/D5558zdepU3njjDV566SX69av4Emh0dDTFxcXk5uayfv16+vXrx/r16xkyZAjR0dEVnjquv/76KsuJiopi27ZtvPnmm7z00ku88847Z1zLhg0b+PXXX0lKSiI4uHpPgNlspnfv3iQnJ3td6Gv0IYiIGZgLjAG6AzeLSPdK2V4CPlBK9QKeBp4DUEolKqUSlFIJwAigEFjhQfvLyT4Ibw+D9S9DQARc8xzctwPu2gDDHzXivntYgLcc28Lk7ydzvPA4b139FlO6TdEir2mU9OzZk5UrV/Lwww+zfv16wsLq9t7Ihg0buPnmmwG45ZZbXMfdQx736dOH5ORkUlKMNXvbt29PQkICAH379iUtLa3GegYNGsSGDRtYt24djz76KOvWrWP9+vVcccUVtbJz4sSJNdbXsWNHlFKsWFGzlNWXEDO16dFfBuxTSh0AEJFFwHhgj1ue7sADzu1EoKoe+/XAUqVU4bmbexbC4+CGD4z4MkGRF6QKdz777TOe3fwsrUNa88ZVb1S7+pFG43HO0vO+UHTu3JmkpCSWLFnCzJkzGTVqFI8//vgZ+dw7OsXFxdWmlVFdyOO0tLQzQhgXFRXVaOcVV1zB+vXrOXjwIOPHj+f5559HRBg3blyN50J52OSykMlV0aJFCxYuXMhVV11FZGQkw4cPrzKf3W5n586d9SJGUW1GBVsBh932053H3NkBlK0GcB0QIiKV1fYm4OOqKhCR6SKyVUS2njhxohYmVUP38Rdc5G0OG8/++CxPb3qay2Mv56OxH2mR1zR6MjIyCAwMZOrUqcyYMYNt27YBFcMXgyGCe/fuxeFwuBYgARg8eDCLFi0CjAHLMqoLeXw2KtfpztChQ1mwYAGdOnXCZDIRERHBkiVLGDx4cJ3KqYnOnTvzxRdfMHXqVLZv335GutVqZebMmbRp04ZevXqdUx2epDZCX5UvovLzyAzgShH5GbgSOAK4mkMRiQF6AsurqkAp9bZSqp9Sql/z5s1rZbg3yCnJ4c6Vd/Jx8sfc2v1W5o6Yq9801TQJdu7cyWWXXUZCQgKzZs3in//8JwDTp09nzJgxrl7t7NmzGTduHCNGjCAmpjzm0+uvv87cuXPp378/OTk5ruOjRo1yhTzu2bMn119/fY3ie9ttt3HnnXeSkJBwRi8/Li4OMAQfYMiQITRr1ozw8PA6lVMb+vfvz3//+1+uvfZa9u/fD8CUKVPo1asX8fHxFBQU8PXXX9e53AtBjWGKRWQg8KRS6hrn/kwApdSZy78Y6cFAslKqtdux+4AeSqnpNRlUX8MUH8g+wD2r7+FowVEeH/g4EzpO8LZJmiaEDlOscaeuYYpr06PfAnQSkfYi4ovhgvmmUgVRIq7J4TMxZuC4czPVuG0aAuvT1zNlyRTyrfm8d817WuQ1Gk2DokahV0rZgLsx3C57gU+VUrtF5GkRudaZbRjwq4j8BrQAZpWdLyJxQBvA+28N1BGlFO/vfp+7V99N65DWLBq7iIToBG+bpdFoNHWiVvPolVJLgCWVjj3utr0YWFzNuWmcOXhb7ym1l/LUpqf4Zv83jGw3kmcGP6PXTNVoNA2Sxvtm7Hlwsugk9yfez44TO/hr779yR+87dNgCjUbTYNFCX4m9p/Zyz+p7yC3N5eUrX2ZU3Chvm6TRaDTnhe6murE8bTnTlk5DRPhgzAda5DUaTaNACz1G5Mk3t7/JjLUz6BrRlY/HfkzXiKqDGWk0mnJee+01Cgvr/rL72WLE1MT8+fPJyMg4p3PnzZvHBx98UGU5cXFxnDx58pztKiM7O5s333yz2nT3a1+yZAmdOnXi0KFDPPnkk7Rq1YqEhAQ6derExIkT2bNnT7Xl1IUmL/SF1kJmrJ3BWzveYkLHCbx7zbtEBZw9QJNGozE4V6E/H85H6O+8806mTZt23uWcjZqEvoxVq1Zxzz33sGzZMtq2bQvAAw88wPbt20lJSeHGG29kxIgRnFe0ACdNWuiP5h/l1mW3surQKh7q9xBPD3oaX7P31oHVaOorBQUFjB07lt69exMfH88nn3zCnDlzXGGKy96Mde+tLl68mNtuuw2A1NRUBg4cSP/+/XnssccqlP3iiy/Sv39/evXqxRNPPAEYsW66devG7bffTo8ePRg1ahRFRUUsXrzYFV648hutmZmZ9O3bF4AdO3YgIhw6dAiADh06UFhYyJNPPslLL71UbTn//ve/6dOnDz179iQ5ORmArKwsJkyYQK9evRgwYAC//GIsM1pWVhnx8fGkpaXxyCOPsH//fhISEnjooYeqvJ/r16/n9ttv5/vvv6dDh6pXsbvxxhsZNWoUH330US3+QmenyQ7G/pz5M/cn3k+pvZS5V81lSKsh3jZJo6kVz//0PMlZyR4ts2tEVx6+7OFq05tymOInnniCSy+9lK+++orVq1czbdq0KuPblDF79mx27dpVbZ6SkhLGjx/PmjVrqo13X0ZZRM/zpUn26L9M+ZI/Lf8TIb4hLBy7UIu8RlMDTTlM8f/+9z+XzSNGjODUqVMV4vXUFYvFwqBBg3j33XdrzOupMMdNqkdvc9h4JekVPtzzIQNiBvDSlS8R5le3H6xG423O1vO+UDTlMMVVia2I4OPjg8PhcB2rfL3VYTKZ+PTTT7n66qt59tlnefTRR6vN+/PPP5/x5HIuNJkefW5pLnevupsP93zIlG5TeOvqt7TIazS1pCmHKR46dKjL5jVr1hAVFUVoaChxcXGu+7Bt2zZSU1NrXW5gYCDfffcdCxcurLZn//nnn7NixQrXk9D50CR69Gk5adyz+h7S89N5cuCTTOo8qeaTNBqNi507d/LQQw9hMpmwWCy89dZbQHmY4piYGBITE11hitu0aUN8fLxLwF9//XUmT57M66+/zqRJ5f9/o0aNYu/evQwcOBAwBnMXLFiA2Wyu1pay8MKV14yFqsMUp6ennzVMsfuasVXx5JNP8sc//pFevXoRGBjI+++/D8CkSZP44IMPSEhIoH///nTu3BmAyMhIBg8eTHx8PGPGjOHFF1+sstyIiAiWLVvG0KFDXWMcr776KgsWLKCgoID4+HhWr16NJ0K31xim+GLj6TDFG49sZMa6GfiID68Of5W+Lfp6rGyN5mKhwxRr3LkQYYobJEopFuxZwF2r7qJlUEs+HvexFnmNRtMkaZSuG6vdyjM/PsMXKV8wos0InrviOR15UqPRNFkandCfKjrFg2seZFvmNqb3ms7fEv6mI09qGgVKqSpnrmiaFufibm9UQv9r1q/cu/peThWf4sWhLzK6/Whvm6TReAR/f39OnTpFZGSkFvsmjFKKU6dO4e/vX6fzGo3Qp+akcsvSWwjxDeH9Me/TI7KHt03SaDxG69atSU9P90jcE03Dxt/fn9atW9ec0Y1GI/RxoXH8Kf5PTOo0ieaB5z8dSaOpT1gsFtq3b+9tMzQNlEYj9CLCnb3v9LYZGo1GU+/Qo5QajUbTyNFCr9FoNI2cevdmrIicAA56247zJAo4/6VqGg/6flRE349y9L2oyPncj3ZKqSoHKOud0DcGRGRrda8iN0X0/aiIvh/l6HtRkQt1P7TrRqPRaBo5Wug1Go2mkaOF/sLwtrcNqGfo+1ERfT/K0feiIhfkfmgfvUZzjohIV2CXUqrRvI+iaZzoHr2mwSMi+W4fh4gUue1POY9yN4vIVE/aqtF4A90T0TR4lFLBZdsikgb8RSm10nsWaTT1C92j9yAi0kZEEkVkr4jsFpH7vG2TtxERs4j8LCLfedmGx0TkgIicFJGFItLMmRYkIotEJEtEskXkRxEJF5GXgf7AO84ng5drUU9bEVniLOs3EbnVLW2wiPwiIlYRsTntGFhd/RfubtQfROQB5//JLhH5WETqFpKxgSMi74lIpojscjsWISI/iEiK89sjvwUt9J7FBvxdKdUNGAD8TUS6e9kmb3MfsNfLNjwEjAKGAK0BK/CqM+0vGE+2rTBeVrkbKFVK/R3YgvF0EOzcr4nPgF+BGGAy8KqIlK1K/QaQBdwFhAPjMe5LlfWfz8U2BESkFXAv0E8pFQ+YgZu8a9VFZz5QOZb6I8AqpVQnYJVz/7zRQu9BlFJHlVLbnNt5GP/IrbxrlfcQkdbAWOAdL5tyB/CIUipDKVUMPAXcKEZgdyvQHOiglLIppbYopQrqWoGIdAJ6A48qpUqUUluB94FbnFnsQC/gS6VUnlJqg1Iq21P1N1B8gAAR8QECgQwv23NRUUqtw2j83RmP8bvB+T3BE3Vpob9AiEgccCnwo3ct8SqvAf8PcHjLAKeYtwGWOF0j2cDPGL/9SOBdYC2wWETSReRZETGfQ1WxwAmlVJHbsYOUN/SzAAUcEZECEVkqIkEerL9BoZQ6ArwEHAKOAjlKqRXetape0EIpdRSMjiMQ7YlCtdBfAEQkGPgcuF8plette7yBiIwDMpVSSd60Qxnzh48AI5RSzdw+/kqpk87e9+NKqa7AUOAPlLsQ6jL3OANoLiIBbsfaOusGOAyEAcOB6cDVlPf+q6u/0eL0PY8H2mM0kkF6htOFQwu9hxERC4bIL1RKfeFte7zIYOBa5yyYRcAIEVngJVvmAbNFpA2AiESLyO+d21eLSHcRMQG5GOMsdud5x4FLalnHPuAX4BkR8RORPsCtwEJn+kAgQym1CcjBcNlcWkP9jZmrgVSl1AmllBX4AhjkZZvqA8dFJAbA+Z3piUK10HsQp5vgXWCvUuoVb9vjTZRSM5VSrZVScRg91NVKKW/12F4AVgKrRSQP2Aj0caa1Ar4G8oBdwBLgU2faq8A0ETktIi+crQLnk8MNQHfgGPAJ8JBSar0zyxVAjIjkA88BXwK7a6i/MXMIGCAigc7/m6vw/qB9feAbjA4Czu+vPVGofjPWg4jIEGA9sJNyv/SjSqkl3rPK+4jIMGCGUmqct23xJiKSgDEw7QscAP6olDrtXau8h4g8BdyI8RTzM8YMpxLvWnXxEJGPgWEYs62OA08AX2E09G0xGsM/KKUqD9jWvS4t9BqNRtO4qZXrRkRGi8ivIrJPRM6Y1ykiQ0Vkm/NFkOsrpdlFZLvz842nDNdoNBpN7aixR++c6vUbMBJIx3iJ5Gal1B63PHFAKDAD+EYptdgtLd/9FXWNRqPRXFxqE+vmMmCfUuoAgIgswpgW5RJ6pVSaM81r86U1Go1GUzW1EfpWGHOAy0gHLq9DHf4ishVjwGW2UuqryhlEZDrG3GKCgoL6du3atQ7FazQajSYpKelkdWvG1kbopYpjdRnBbauUyhCRSzCmt+1USu2vUJhSb+MMuN+vXz+1devWOhSv0Wg0GhE5WF1abQZj0zFeIS+jNXWISaGUynB+HwDWYIQF0Gg0Gs1FojZCvwXoJCLtRcQX4+WXWs2ecYZ79XNuR2G8Lbnn7GedB7u+gNKmEg9Ko9FoakeNQq+UsmGETl2O8ebap0qp3SLytIhcCyAi/UUkHSNOx/+JyG7n6d2ArSKyA0jE8NFfGKE/uQ8W/xHe6A87F4N+P0Cj0WiAevjC1Hn56A9thqX/D47ugLYDYczzENPbswZqNF7AarWSnp5OcXGxt03ReBl/f39at26NxWKpcFxEkpRS/ao6p3EJPYDDDj8vgFVPQ+Ep6DMNrnocgqI8Z6RGc5FJTU0lJCSEyMhIjNAwmqaIUopTp06Rl5dH+/btK6SdTegbX1Azkxn63gr3JMGAu2D7QpjTBza9CXart63TaM6J4uJiLfIaRITIyMg6P9k1PqEvI6AZjH4O7toIrfvC8pnw1mDYv9rblmk054QWeQ2c2++g8Qp9Gc27wNQv4KaPwV4KH14HH0+GrAPetkyj0WguCo1f6AFEoOvv4G8/wlVPwIE1MPdyWPkUlOR72zqNpt6TnZ3Nm2++6dEyn3zySV566SWPlqmpmqYh9GX4+MEVDxr++x4T4X+vwBv9YMcnejqmRnMWLoTQny92e1NYiMsz1CYEQuMjNAYm/h/0/7MxHfPL6bDlHWM6Zqs+NZ+v0XiRp77dzZ4Mzy5F3D02lCd+36Pa9EceeYT9+/eTkJDAyJEjiY6O5tNPP6WkpITrrruOp556CoAJEyZw+PBhiouLue+++5g+fToAy5Yt49FHH8VutxMVFcWqVasA2LNnD8OGDePQoUPcf//93HvvvQAsWLCAOXPmUFpayuWXX86bb76J2WwmODiYBx98kOXLl/Pyyy8zZMgQj96HxkrT6tFXps1l8JfVcO0bcDoV/jMCvr4b8k942zKNpl4xe/ZsOnTowPbt2xk5ciQpKSn89NNPbN++naSkJNatWwfAe++9R1JSElu3bmXOnDmcOnWKEydOcPvtt/P555+zY8cOPvvsM1e5ycnJLF++nJ9++omnnnoKq9XK3r17+eSTT9iwYQPbt2/HbDazcKGx9G5BQQHx8fH8+OOPWuTrQNPs0btjMkGfW6D7tbD2BfhxHuz5Gq58GC6bDj6+3rZQo6nA2XreF4MVK1awYsUKLr3UCFuVn59PSkoKQ4cOZc6cOXz55ZcAHD58mJSUFE6cOMHQoUNd874jIiJcZY0dOxY/Pz/8/PyIjo7m+PHjrFq1iqSkJPr37w9AUVER0dHRAJjNZiZNmnQxL7dRoIW+DP8wuGYW9L0Nlj0CK/4BSfNh9GzodLW3rdNo6g1KKWbOnMkdd9xR4fiaNWtYuXIlmzZtIjAwkGHDhlFcXIxSqtopgX5+fq5ts9mMzWZDKcWtt97Kc889d0Z+f39/zGazZy+oCdC0XTdVEdUJpiyGmz8BZYeFk+Cjm+DU/prP1WgaKSEhIeTl5QFwzTXX8N5775Gfb8xYO3LkCJmZmeTk5BAeHk5gYCDJycls3rwZgIEDB7J27VpSU1MByMo6+1rXV111FYsXLyYzM9OV/+DBaiPwamqB7tFXhQh0GQ0dhsPmt2Ddi/DmABjwVxg6A/xCvG2hRnNRiYyMZPDgwcTHxzNmzBgmT57MwIEDAQgODmbBggWMHj2aefPm0atXL7p06cKAAQMAaN68OW+//TYTJ07E4XAQHR3NDz/8UG1d3bt355lnnmHUqFE4HA4sFgtz586lXbt2F+VaGyONL9bNhSDvmDHnfsdHENwSrn4Set1o+Pc1movA3r176datm7fN0NQTqvo9NIlYNw7l4Pmfnmff6X2eLzykJVz3FvxlFYS1gq/uhHdHwpEkz9el0Wg0HqbRCP3hvMN8e+Bb/vDtH3g16VUKrYWer6R1P/jzSpjwFmQfMqZjfvU3yDvu+bo0Go3GQzQaoW8X2o5vJnzDuA7jeG/Xe0z4egKrD12AAGYmEyRMNt6uHXQv/PIJ/LsvbJgDtlLP16fRaDTnSaMReoAI/wj+NfhfvD/6fYIsQdyXeB/3rLqHI/lHPF+ZfyiM+hf8dTO0GwQ/PAZvDYTfVni+Lo1GozkPGpXQl9GnRR8+/f2n/L3v3/nx2I9M+GoC7+x8B+uFiEcf1RGmfAqTnW/7ffQHWPgHyLkAjYtGo9GcA41S6AEsJgu3xd/GNxO+YUirIby+7XUmfTuJn47+dGEq7DwK7toEo56BtA0wbzAkf39h6tJoNJo60GiFvoyWQS15dfirzL1qLqX2Uv684s/MXD+Tk0UnPV+Zjy8MugfuWAfN2sKiybDkIbDqdT41jZPXXnuNwsK6T3wIDg4+5zrnz59PRkbGOZ07b948PvjggyrLiYuL4+TJs+vCmjVrGDdunGv/n//8J9dccw0lJSUMGzaMLl260KtXL7p27crdd99Ndnb2OdnpaRq90JcxtPVQvhr/FdN7TWdZ2jKu/fJaFiUvwu64AKFOozrCn3+AgXfDT28bs3Mykz1fj0bjZc5V6M+H8xH6O++8k2nTpp13OQCzZs1iw4YNfPXVV65QDgsXLuSXX37hl19+wc/Pj/Hjx59z+Z6kSb0Z6+/jzz2X3sO4S8Yxa/MsZv04i6/2fcVjAx6jR5SHA0X5+Bmxcy4ZBl/eCW8PgzGzoc+txpu3Gs25svQROLbTs2W27Gn8PquhoKCAG264gfT0dOx2O4899hjHjx8nIyOD4cOHExUVRWJiIsHBwa7QCIsXL+a7775j/vz5pKamMnnyZGw2G6NHj65Q9osvvnhGyOO0tDTGjBnDkCFD2LhxI61ateLrr7/m+++/Z+vWrUyZMoWAgAA2bdpEQEAAAJmZmYwZM4akpCR27NhBQkICBw8epG3btnTo0IGdO3fywgsvEBwcTFxc3BnlAPz73//m22+/xWq18tlnn9G1a9cq78fLL7/MkiVLWL58uat+d3x9fXnhhRfo2LEjO3bsoHfv3uf0Z/EUTaZH7077sPb8Z9R/eP6K5zleeJybv7+ZWZtnkVvq2RjfAHQaaaxb2/Zy+PY++OxWKDrt+Xo0mgvIsmXLiI2NZceOHezatYvRo0dz7733EhsbS2JiIomJiWc9/7777uOuu+5iy5YttGzZ0nV8xYoV1YY8TklJ4W9/+xu7d++mWbNmfP7551x//fX069ePhQsXsn379goiGx0dTXFxMbm5uaxfv55+/fqxfv16Dh48SHR0NIGBga681ZUTFRXFtm3buOuuu6pd/WrDhg3MmzePpUuXntUFZTab6d27N8nJ3n+ab1I9endEhN9d8juuaH0Fb/z8Bot+XcQPB39gRv8ZjG0/1rMLMYe0gKlfwsY5sPpfcGQbTHrXEH+Npq6cped9oejZsyczZszg4YcfZty4cVxxxRV1On/Dhg18/vnnANxyyy08/PDDQPUhj9u2bUv79u1JSEgAoG/fvqSlpdVYz6BBg9iwYQPr1q3j0UcfZdmyZSilam3vxIkTXfV98cUXVebp2LEjp0+fZsWKFVx//fVnLa8uIWaUUigUJvF8/7tJ9ujdCfENYeblM/lo7EfEBMUwc/1Mbl9xOwdyPLx4uMkEQ+6HP60Akxn+O8YIlnYhxgg0mmootZeSVZzFkbwjHC88Tm5JLqX20hoFqXPnziQlJdGzZ09mzpzJ008/XWU+9w5ScXFxtWlllIU83r59O9u3b2ffvn38+c9/BqoOYVwTV1xxhasXP378eHbs2MH//vc/hg4dWuO57nWerb4WLVqwZMkSHnjgARITE1FKYXfYUUpRYiuh0FpIXmkeWYVZ7PhlBy3btySzMJNjBcfIyM/gcN5hDuYeJDUnlX3Z+/jt9G8kZyWz99Re0nLTamVnXWnyQl9Gj8geLPjdAh4b8Bh7svYw6ZtJzNk2hyJbkWcrat0X7lgPPa6D1c/AB+Mh99wHhDSas2F32MktySUjP4OU0ymknE7haP5R8qx5nCw8yeG8w6ScTiE5K5m0nDSOFRwjuzibYltxBfHPyMggMDCQqVOnMmPGDLZt2wZUDF8Mhgju3bsXh8PhWoAEYPDgwSxatAjAtVoUVB/y+GxUrtOdoUOHsmDBAjp16oTJZCIiIoIlS5YwePDgKsvJzc3F6rBSZC1CocgrySO7OJvs4mysDivHC46TkZ9Bel46h3IPcSz/GIXWQqS58Op/X+WmyTfxxZovSM5KptBWSHp+Oqk5qew/tZ8ZD8+geUxzoi6J4kThCU4XnyavNI9iWzE2hw1B8DX5EugTSKhvKJEBkYT7hdf0Jz0nmqzrpirMJjM3dLmBEW1H8MrWV/jPzv+wJHUJMy+byZVtrvRcRf6hMOkd6HgVfD8D3hoE49+Err/zXB2aJolSiiJbEfnWfPKt+RRZjY6KSUwEWYKI8I8g2BKMr9kXhaLYVkyxvdj4thWTVZzlEngRwc/sR4BPABu3buTJfzyJj9kHi8XCW2+9BcD06dMZM2YMMTExJCYmMnv2bMaNG0ebNm2Ij493Cfjrr7/O5MmTef311yusEDVq1Cj27t17bDLGGwAAHgdJREFURsjjsy0uctttt3HnnXeeMRgLxhRJwNWDHzJkCOnp6YSGhVJkLaLYVgylkJ6Xzug/jObP0/+Mn78fC5cuxOawcST/CIV+hf+/vTMPj6pK8//n3FtbqrKQBcISwpKgaIeI2jgSW0dBHbRRwWVEH0dHbTcchrZte7Qfn5+2o7ZObzDdrTYKjbS4MGAHRBoZonbruIAiDbJ0k4BICAETSEgqSa3n98epqlRloyCVVCjO53nuc88599xbJzdV3/ec9577Hurb6vEGvNS31mMYBoZQm0QtouKwOCg7v4z5v5vP3Fvn8se1f8Ru2nl09qPY7Xa8Hi9Tpk7h7bfeJjc7F0MYiXUHHyc6THEPbKzdyJOfPMnuxt1MGTmFh897mGHpwxL7IXWVsPx2qN2ili687D/B6kjsZ2g46j2Kx+8hLy0vqT+4E6WnMMXegJdmrxJ2t89NUAYBSLOk4bK5SLemk2ZJi8v3K6XEE/B0MgAB2e5itJk20ixpOCwOHKYDh8WBxUh+nzEog3gDXjwBD96AV6WDKt1xGrXVtGI37dgMGzbThtWwYhomhjAwhRkRdoEYkN+X4w1TrIX+GPgCPpZsX8ILf30BIQT3nXUft5x5C1bDmrgP8XtUvPtPfgv5JXD9Ihh8euKufwrT6Glk0ZeLeG3na7T6W8m0ZVI8qJjiQcUUDSpS6exichw5x75YEon+YQeCAdw+d6TXHg7tYTWspNvScVlduKyuhImvlBJf0Bcj/q3+VvzBdh+21bAq4Y8Sf6thTbhIBmUQX8CHN9hB0AOemPYAWAwLNtOmBN20YTNU2mpa++SBZ3+ihb6PqGmu4ZkNz/DevvcoHlTMo+c/yrn55yb2Q/6+DsrvA68brngWzrlVz7k/QZq9zfxhxx9Ysm0Jbp+bK8deyYS8CVQ1VFHVUMWuhl00edv9vDmOHIoGFVGUVcS47HERI5Blz0riX6HwBX1s376dYWOHdemOcVldpNvSsRm2fu19+oP+TuLvDbRHcDWF2Un87ab9mG0MG5ZI7zzojQh69PVBuVujRdxm2iJ500jdtWX7ROiFENOA+YAJvCSlfKbD8YuAeUApMEtKuTzq2G3Ao6Hsk1LKl3v6rIEq9GHe3/c+P/30p9S4a7i66Goe/PaDie0NNtXCm3fDnj+rB7bT50HaoMRdP8Vp9bfy+s7XWfTlIho8DUwtnMr9E+9nXPa4mHpSSr5p/YbKhkoqj1RS1VhFZUMlVQ1VuH3uSL28tLxOI4CiQUVk2PpuOUkpJV83fc1HNR/xcc3HbKjdwFPjnmLomKGkWdNIt6pee7zumP4kEAzEuH5a/a14Ap4Yv39Y9B0WBzbDhi/oa++dh0Q9WpcMYbQLuGnDbrQL+kBwGSWDhAu9EMIE/g5cBlQDG4GbpJTbo+qMBjKBHwKrwkIvhMgBPgO+DUjgc+BcKWW3bwwNdKEHaPG18OLWF1m8bTFOi5Pvn/t9rht3XeJ+dMEgfDRfzcrJGK4e3Oo59z3iDXhZsWsFC7YsoK61jgtGXMCciXOO+41nKSW17tqI6O9q2EVVQxW7G3fHzMDKd+bHun9CaafV2cPVu6fR08inBz7lo5qP+OTAJ5HQ2iPSR1A2vIzp6dMp/VbpSSlsYd95m7+N1kArHr+HVn9r5FkCKAMQ9pdHBD3kQ7cYlgHpJ08mfSH0k4HHpZT/FMo/AiCl/GkXdRcDq6OE/ibgYinlPaH874D3pZSvdfd5J4PQh9ndsJsnP32SjbUbKc0r5dHzH+WM3ASu61n9GSy/Axqr4ZJH4Ds/AMPEH/TT4m+hxdeC2+dWe787Jt/ib8Fm2ri08FLyXfmJa9MAwx/081bVW7zw1xeocddwbv65zDl7TsLdakEZpKa5Ro0AQkYgvHmD7e6E4a7hFGfHGoAxWWNIs8S+Ju8L+tjyzZZIr31b/TaCMki6NZ3zhp7H5OGTKRtexsiMkQghUm7N2LB7xhfwYTWtfeLPT2WOV+jj6R6MAPZF5auBeLuXXZ07omMlIcTdwN0AhYWFcV46+YwdNJaFly9k9e7V/PyznzPr7VncPP5m7p94P+k29Wq0lBJv0NsuyD43rf5WlferfExZWKzDQl5cQssRB+6/L6Jl9yu0mBY8wfhXsvqvjf/F5OGTmVE8gykjp2AzbX11O/qVoAzyzlfv8Nzm5/jq6FeU5Jbw2OTHmDx8cp8IhiEMCjIKKMgo4OKRF0fKA8EA1c3V7S6ghioqGyv5qOajyMNBgaAgo4CiQUWMyRzDnsY9bKjdQIu/BUMYTMibwD2l91A2vIySvJKTstd+vAghIj13Td8Tzzeqq19NvE9w4zpXSrkAWACqRx/ntQcEQgiuKrqKiwou4tdf/JqlO5aysmolLqtLCbivFb889ht9ABZhwWl14rQ6cVnUQ7Y0m5OcEZNxNh3EVf0ZTkyc46fjGjYRp8WJy+pS50SlXVYXTouTutY6VlatZFXVKh7680Nk2bO4csyVzCyemdiRRz8ipeS9fe/xm82/YdeRXYzLHsf8S+ZzychLktIjNA2TUZmjGJU5iqmFUyPlvqCPfUf3dXIBfVD9AcNcw5g+djplw8uYNGwSmbbMfm93opg3bx533313TByZeIgOfna8LF68mMsvv5zhw4cf97kvvPACTqeTW2+9tdN1woHO8vLyTqhdYRoaGnj11VeZPXt2l8ej//Y1a9Ywd+5cKioqWLRoES+++CKDBw/G7XYzYcIEnnzySc4888xetQfiE/pqYGRUvgCI91XOauDiDue+H+e5JxVZ9iwePf9RZhTP4LWdr2EIo5MQh0XYZXFFBD1cx2V1HXv4WrdLzbn/4CX4h3vh0p/0OOe+0FrInLPnMPus2Xx64FPKK8tZ8fcVvLbzNU7PPp0ZxTP47tjvku3om7fxEomUko8PfMxvvvgNW+u2MipzFM9e+CzTxkwbcA8kQU03HDtoLGMHjY0pDwQDKTUbZN68edxyyy3HLfS9YfHixZSUlJyQ0N97770JuU5PNDQ08Nxzz3Ur9GEqKiqYM2cO69ati3gyHnjgAX74wx8C8MYbbzBlyhS2bt3K4MGDe9WmeIR+IzBOCDEG2A/MAm6O8/rvAE8LIcJKcjnwyHG38iSiJK+Ep77zVN9cPG8cfK8C1j8OnzynVrK6fhEMPq3H00zDpGxEGWUjymj0NPKnPX+ivLKcZzc+yy8+/wUXF1zMzHEzKRteNiDdBpsObuK/v/hvPj/4OcNcw3ii7AmuKrpqQLb1WCRC5J/d8Cw7Dyc2IuL4nPH8x3n/0e3xUzlM8eHDh7njjjvYvXs3TqeTBQsWUFpayuOPP056enpEmEtKSli9ejUPP/wwVVVVTJw4kcsuu4yf/exnne7nBx98wF133cWaNWsoKirq8p7feOONvP3227z66qvMnTv3+P+pURyzKySl9AP/hhLtHcAyKeU2IcQTQoirAYQQk4QQ1cANwO+EENtC5x4G/hNlLDYCT4TKNCeKxQ7Tfgo3L4OmGljwj7BpCcT5PkSWPYtZ42fx+vTXWXH1Cm4afxObDm3i/or7uXz55fzy818mPqDbCbKtfhv3rr+X29bext6je3nkvEdYPXM1M8fNPClF/mTmVA5T/Nhjj3H22WezZcsWnn766cjCJd3xzDPPUFRUxObNm7sUeY/HwzXXXEN5eXm38e7DnHPOOQkJcxzXr0VKuQZY06Hs/0WlN6LcMl2duwhY1Is2arritH9Sce7fvBtWzYGq92D6r45rzv1p2afxo0k/4oFzHuAv+/9C+a5ylmxbwu+//D1nDT6LGcUzmDZ6WuTBcn9ReaSS327+Leu/Xk+WPYsfnPsDZo2f1WnmyqlKTz3vvuJUDlP84YcfRto+ZcoU6uvraWxsjP+P74DVaqWsrIyFCxcyf/78Husm6oXWgefc1MRPxlD4l3KY+hhsXwm/uxD2Hf/i51bTytTCqfx66q9Zf8N6Hjz3QZq8Tfzk459wybJL+PEHP2bDgQ0x8577gq+Pfs3DHzzMtauu5eMDHzP7rNmsvXYtt5fcrkU+yZzKYYq7ElshBBaLhWCw/TfR8e/tDsMwWLZsGRs3buTpp5/use4XX3yRkGm1WuhPdgwDLvwB3PGOyi+aBn/5+QnHuc9Ly+NfS/6V8mvKWXrlUq4quor39r3Hnevu5Mo3r+T5vz5PTXNiwyrXumt5/KPHubr8air2VnB7ye2svXYt9028r99HE5quSdUwxd1dp+M1w21+//33ycvLIzMzk9GjR0fuw6ZNm9izZ0/c13U6naxevZqlS5eycOHCLuusWLGCdevWcdNNNx2zjcdCOzpThZGT4N4P4a3vq1Ws9vwZZi6AzBOLtimEoHRwKaWDS3lo0kNUfF1BeWU5z21+juc3P895w85jZvFMphZOxWE5sWibda11vLT1JZb9bRkAN55+I3eV3kVeWu+mt2kSz9atW3nooYcwDOOkDFOcnd15ZlnH63TH448/zu23305paSlOp5OXX1ZRXK677jqWLFnCxIkTmTRpEqedpiZF5ObmcsEFF1BSUsIVV1zRpZ8eICcnh7Vr13LRRRdFpnT+6le/4pVXXsHtdlNSUsK7777b6xk3oIOapR5SwualsOYhMCwqXk7pjVA4WfX+e8n+5v2sqlzFyqqV7G/eT4Y1g2ljpjGzeCYleSVxzWWPjijpDXiZUTyDe0rvSXwI6BQi1d6M1fQOHb1So6jbpZYq3LEafG7IGgkTblCiP6TnJ/3xEJRBNtZupLyynPV719MWaKMoq4gZxTOYXjS9y155x4iSV4y5gtkTZzMqc1Sv25PqaKHXRKOFXhOL1w0718CWN6DqXZABGFqqBH/C9eqBbi9p8jbxzlfv8MfKP7Llmy2YwuTCgguZUTyDiwouwh/088bON1j45cJIRMnZE2dzWnbP8/817Wih10SjhV7TPc2H4Ms3lejXbAJhwJh/VKJ/xnSw9z707u6G3ZRXlvPW7reoa60jx5GDIQwVUXL4Bcw5+/gjSmrUD3v8+PE68JcGKSU7d+7UQq+Jg7pdsGWZEv2GvWBJg/HfVaJfdAmYvVtByx/083/7/4+VVStp87dx54Q7E79QyynEnj17yMjIIDc3V4v9KYyUkvr6epqamhgzZkzMMS30mu6RUs293/IGbHsTWo+AMw9KrlOiP+IcvcrVAMDn81FdXR33XG1N6uJwOCgoKMBqje2MaaHXxIffC5Xrlej/7U8Q8EBOkRL80hsgZ+yxr6HRaJKCFnrN8dPWCNtXKdH/6kNAQsF5UPrP8K1rwZWb7BZqNJootNBrekdjNWxdrkT/0HY1P7/4MiX6p18BVh2eQKNJNlroNYmj9ksl+Fv/B5oOgC0DzrxGif7o78BAjbUe/p7r5w2aFEULvSbxBAPKpbNlmQqo5m1SC5lPuF759IeWJP4zAz7lUmptUPu2Ix3yDT3nTRukD4H0fMjIh/ShUenQljFUPYw2dXQQzcmFFnpN3+JrVQ9vtyyDyv+FoB+GfEv18idcD1mhCNZSgqcpPlHuKu9r6bkdpg0cg1SoZkeWSjuyVN6eCQGvepeguRaaDkLzQXXdTghwDe7aCKQPCRmIISpvcyX8dmo0J4IWek3/4a5X0zS3LIPqDYCAQSPbBf5YoY7tWZCW1VmoHYM65DsezzqxZwW+NnAfahf+aCMQ3poOqjrBLsLk2jLaRb+jEYg2EGk5CYk1pNF0hxZ6TXI4vBu2/A/U7+pamDvm7ZkD18cfDELrYWiq7WwEIulaNWLwdhGi1rCAa4gaITiywJauRgORLSMqHTpmT4/Nh9O9fJmtXwgGwd+mNl9r93sZUMYxawRkDDs5/rYBSk9Crx2Rmr4jZyxc3P+rIfUJhgGuPLVxjOcPXnf3RqC5Vo1u3HVq73Wrzd8af1tMexcGwKVCWMQYj2hD0cFYWNMg6FMjGn9rD/vWKGE+Vt2ofcBzAjdZqNFP5nDIHKFcfpkjVD6cTs/Xz09OAH3HNJpEY3MpI3c8L5gFAyHRb+6wd8cahMixLuq562Lzx2M8esKSBlZH13tnLlgcynBY03qu29UeoYxf4344ur99/81OqKxQkVejEWbIGIxQo4DMEVHpAmUU0ocM3JFhktBCr9EMBAwTHJlqSxTBQKzwh9OeZiWgpi0kuD2IscXeD1NSuxkhSakelh+tCRmA6tC+RqUPbFGTAPwdwkIYFjUDLHN4uzHIKogdKTjzBsYzk2BQua+C/vZnQAkILtgRLfQaTapimKFnIVnJbsmJIQSkZastv5uIp1Kq+EyN1WokED0qaNwP+zepNRk6upJMm3omEHYJOXPURIGw4AYDHfZRadnxWBd1O9UJp6PzfqDDM9KCSfC99Qm/lVroNRrNyYsQSqSdOTCstOs6Uiq3VowhqG4fKez7RE3jNUw1GghvwojNG2bnOhbHMeqE0qKLso57YapRRx+ghV6j0aQ2QkD6YLUNn5js1iSFAeCk0mg0Gk1fooVeo9FoUhwt9BqNRpPiaKHXaDSaFEcLvUaj0aQ4Wug1Go0mxdFCr9FoNClOXEIvhJgmhPibEKJSCPFwF8ftQog3Qsc/FUKMDpWPFkK0CiE2h7YXEtt8jUaj0RyLY74wJYQwgd8ClwHVwEYhxCop5faoancCR6SUxUKIWcCzwI2hY1VSylPzLQWNRqMZAMTToz8PqJRS7pZSeoHXgWs61LkGeDmUXg5MFUIvzqnRaDQDgXiEfgSwLypfHSrrso6U0g80ArmhY2OEEF8IIf4shLiwl+3tkT11bgbaQioajUaTbOKJddNVz7yjmnZX5wBQKKWsF0KcC5QLIb4lpTwac7IQdwN3AxQWFsbRpM7UNXuY8ov3GZ6VxpTxQ5h6xhDOH5uLw6rjUms0mlObeIS+GhgZlS8AarqpUy2EsABZwGGputceACnl50KIKuA0IGatQCnlAmABqKUET+DvwGE1eebaCazfcYjln1fzh0/24rSZfKc4j0vPyOeS8UMYnGE/kUtrNBrNSU08Qr8RGCeEGAPsB2YBN3eoswq4DfgYuB54V0ophRCDUYIfEEKMBcYBuxPW+ijS7RZunFTIjZMKafMF+Hh3PRU7DlKx4xDrth9ECDirYBCXnjGEKePzOWNYBvoxgkajORWIa3FwIcSVwDzABBZJKZ8SQjwBfCalXCWEcAB/AM4GDgOzpJS7hRDXAU8AfiAAPCalfKunz0r04uBSSrYfOErFjkNU7DjIX6sbARgxSLt4NBpN6tDT4uBxCX1/kmih78ihpjbe23mI9TsO8eGuOlp9AZw2kwvH5TF1vHbxaDSakxMt9N3Q5gvwcVU963cc5N2dhzjQ2Bbj4pl6Rj7jh2oXj0ajGfhooY8DKSXbao7y7s7uXTyTi3KxW7SLR6PRDDy00J8Ah4628W7YxVP5DW2+YLuL54x8powfQl66dvFoNJqBgRb6XhLt4qnYcYjao8rFM3HkIC4Nib528Wg0mmSihT6BhF08FTsOUbHzIFuiXDxTQ37988fmYLeYSCnxBoL4AhKvP4gvEMTrD4bKgpEyj7/rOuF8e1lsna7qegNBfH71uVJK0mwm6XYLTpsFl91Cut3EabOoMrs65rK1p8PHXKF6pnFyGi8pJb6AxOMP4PWre5zhsJDhsCa7aRpNn6CFvg85eDRqFk/IxWMxBEKAL5D4e2szDWwWA6spQnuVby9XaavFwBDQ4gnQ7PHT4vXT7AnQ4vXT4g3E/XlpVhOX3cQVMgjdp0P56PLQMYsplEEKCa7aBzrlPTH5sBEL4PEpAxbZdzq3fR99rCtyXDYKc5yMznVSmOtidK6TUblOCnNc5KXb9KhMc9Kihb6faPMF+Kiqjg17jiAEWE0De1iUQ+JrixJma7Q4WwQ208RqEZFjdktsHaspEiJEgaCMCH6zxx8xBm6PH7fXjztkENrLAmrvCUTVCeVD6WCCv0a20N9vC232yN7skDewWcwOeQO7aWC3muo6VnW/G1p97K13s7e+hb31LdQ0thL99XfZzIj4F+Y6GZ3rYlSOSg/LSjtpRjctXj+1jW3UHm3j4NE2ahs9HAynj7ZxsLGNumYvNosRMcrhkV26o31E57JbSA+PBCPlalQYPidcZjWTv7SFlBKPP0ibL0CrL0CrV+3bfAFavUFV5gvQFioP12nzBZDAIKeVHKeNbJeNHJeNbKeNbKeVQU7bSfG/10Kv6VPCP7CIYYgyAG5PAH8w2C7aphkR3vZ9rCDbTAOjH35YHn+A6iOtMeK/t97N3sMt7DvcEjMis5kGBTlpjM51RUYEo3JdFOY6GZntxGbpe6ELBCV1zZ6IiB8KCXdYyMMi3uTxdzo33W4hP9PO0CwH+ZkOBqfb8QaCkf9R+H/X7PHH/B+9ga5HRh2xWYyIgUi3WyPGoLOxiDUsaTYTj//YQtxJuCPHgu1pf4ATkTOH1UBKuh0FCgFZae1GINtpI8dlVQahY5lTGYlMh7VfvsOx7dRCr9EcF4Gg5EBja6wBqG9h72GVjnZ/GQKGZaUxOk+5gKLdQaNynbjsx4400tTm4+DRkGDH9MbbONjk4WBjG980ewh0GDqZhmBIhp38TIcS8kwH+VkOhmaqbUimg6FZDtLjaENXeP3BTgagOWp017lcuQjDI7/mttiR4fFgMw0cVoM0m0ma1cRhNSPpcF6VGe1lUcfTbKHjUed1vIbd0t6paPUGONLi5bDb2753eznc4uNIqEyVq/xht7dbQ2gI1IggYgysUaOE8Kih3TBku2xk2C29GrFroddoEoiUkrpmb9RIwB0yACp9pMUXUz8v3R5xBxXmOPH4gxwMiXm4F96VCGY4LEqwQ73wsIjnZ6ie+dBMB7np9pPCrQAQDEpafMoINLUp8W/zBWLEN5x2WAwsA8Ad1BNSSlq8AQ67vTS0+Djc4o0YgI4Go6HFF8l39+zOYgjOH5vLK9/7hxNqT09Cf2JmXqM5hRFCMDjDzuAMO98endPpeGOrj6/rW9h72B0zGviosp43j+7HYohID/z0/AwuGjc4Itz5EWG347Sl1s/TMETEr5+fmezW9B4hRMQ9NbLz16BLpJQ0e/wccXc2DEdavGQ7bX3S1tT6Jmk0A4CsNCsTCrKYUJDV6ZjHH8Bq9M8zCM3AQwhBhsNKhsNKYa6z3z5XC71G04/oEBqaZDCwnWAajUaj6TVa6DUajSbFGXCzboQQ3wB7k92OXpIH1CW7EQMIfT9i0fejHX0vYunN/RglpRzc1YEBJ/SpgBDis+6mOZ2K6PsRi74f7eh7EUtf3Q/tutFoNJoURwu9RqPRpDha6PuGBcluwABD349Y9P1oR9+LWPrkfmgfvUaj0aQ4ukev0Wg0KY4Weo1Go0lxtNAnECHESCHEe0KIHUKIbUKIucluU7IRQphCiC+EEKuT3ZZkI4QYJIRYLoTYGfqOTE52m5KJEOKB0O/kSyHEa0IIR7Lb1J8IIRYJIQ4JIb6MKssRQvyvEGJXaJ+diM/SQp9Y/MCDUsozgPOB+4UQZya5TclmLrAj2Y0YIMwH1kopxwNncQrfFyHECODfgW9LKUsAE5iV3Fb1O4uBaR3KHgYqpJTjgIpQvtdooU8gUsoDUspNoXQT6oc8IrmtSh5CiALgu8BLyW5LshFCZAIXAQsBpJReKWVDcluVdCxAmhDCAjiBmiS3p1+RUv4FONyh+Brg5VD6ZWBGIj5LC30fIYQYDZwNfJrcliSVecCPgPjWo0ttxgLfAL8PubJeEkK4kt2oZCGl3A/8HPgaOAA0SinXJbdVA4J8KeUBUB1HYEgiLqqFvg8QQqQDK4DvSymPJrs9yUAIMR04JKX8PNltGSBYgHOA56WUZwNuEjQsPxkJ+Z6vAcYAwwGXEOKW5LYqddFCn2CEEFaUyC+VUr6Z7PYkkQuAq4UQXwGvA1OEEK8kt0lJpRqollKGR3jLUcJ/qnIpsEdK+Y2U0ge8CZQluU0DgYNCiGEAof2hRFxUC30CEWpl34XADinlL5PdnmQipXxESlkgpRyNesj2rpTylO2xSSlrgX1CiNNDRVOB7UlsUrL5GjhfCOEM/W6mcgo/nI5iFXBbKH0bsDIRF9UrTCWWC4B/AbYKITaHyn4spVyTxDZpBg5zgKVCCBuwG7g9ye1JGlLKT4UQy4FNqNlqX3CKhUMQQrwGXAzkCSGqgceAZ4BlQog7UcbwhoR8lg6BoNFoNKmNdt1oNBpNiqOFXqPRaFIcLfQajUaT4mih12g0mhRHC71Go9GkOFroNRqNJsXRQq/RaDQpzv8HBObuwkisLiUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = 10\n",
    "x = list(range(1, epochs+1))\n",
    "\n",
    "plt.subplot(2, 1, 1)\n",
    "plt.plot(x, [teacher_history[i][1] for i in range(epochs)], label='teacher')\n",
    "plt.plot(x, [student_kd_history[i][1] for i in range(epochs)], label='student with KD')\n",
    "plt.plot(x, [student_simple_history[i][1] for i in range(epochs)], label='student without KD')\n",
    "\n",
    "plt.title('Test accuracy')\n",
    "plt.legend()\n",
    "\n",
    "\n",
    "plt.subplot(2, 1, 2)\n",
    "plt.plot(x, [teacher_history[i][0] for i in range(epochs)], label='teacher')\n",
    "plt.plot(x, [student_kd_history[i][0] for i in range(epochs)], label='student with KD')\n",
    "plt.plot(x, [student_simple_history[i][0] for i in range(epochs)], label='student without KD')\n",
    "\n",
    "plt.title('Test loss')\n",
    "plt.legend()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}