cnn_lstm_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cnn_lstm_cnn.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "TqN5BDW-BWb5",
        "zkaNR3VhBoPw"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "8GrIEHEhAKcZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Author: Qiu Yang\n",
        "\n",
        "CNN-RNN-CNN\n",
        "\n",
        "**Settings used for current result:**\n",
        "\n",
        "toy_dataset, 20000 training , 4000 testing, [X, Y, 0/1], added 0s, model_2, 87%"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "MtsFQfHzulET",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import torch.nn as nn\n",
        "import math\n",
        "import time\n",
        "import numpy as np\n",
        "import random\n",
        "import pickle\n",
        "import os.path as path\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.path import Path\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "ZmsvnbXRulEZ",
        "outputId": "c7333664-6d0a-4e46-d04a-88f15010cfb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "device= torch.device(\"cuda\")\n",
        "#device= torch.device(\"cpu\")\n",
        "print(device)\n",
        "torch.cuda.is_available()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "metadata": {
        "id": "suS_js-gABnF",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Datasets"
      ]
    },
    {
      "metadata": {
        "id": "ZqIfo7oPm20Y",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**unpack dataset**"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "jtkWxu9lulEh",
        "outputId": "6989231b-cbfb-491a-b2dd-a96da3b3f3a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "data_path  = ''\n",
        "with open(path.join(data_path,'data_X'),'rb') as f:\n",
        "    X=pickle.load(f)\n",
        "    \n",
        "with open(path.join(data_path,'data_Y'),'rb') as f:\n",
        "    Y=pickle.load(f)\n",
        "    \n",
        "len_train_X=int(len(X)*0.8)\n",
        "\n",
        "train_data_raw=np.array(X[:len_train_X])\n",
        "train_label=np.array(Y[:len_train_X])\n",
        "test_data_raw=np.array(X[len_train_X:])\n",
        "test_label=np.array(Y[len_train_X:])\n",
        "\n",
        "\n",
        "print(len(train_data_raw),len(train_label))\n",
        "\n",
        "print(len(test_data_raw),len(test_label))\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "35685 35685\n",
            "8922 8922\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "aKZzEEoSZPal",
        "colab_type": "code",
        "outputId": "4eb7f584-28f8-480b-f9d9-ec504e660134",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "#check classes\n",
        "a = set(train_label)\n",
        "print(a)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'penguin', 'calendar', 'teddy-bear', 'snowman', 'blackberry'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "kQq4PTdismBO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**A random sample**"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "1QvGLEOiulEn",
        "outputId": "5cdc0120-d351-48da-f56c-e703f4340b00",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        }
      },
      "cell_type": "code",
      "source": [
        "sketch = random.choice(train_data_raw)\n",
        "for i in range(len(sketch)):\n",
        "    plt.plot(sketch[i][0][:], sketch[i][1][:])\n",
        "    \n",
        "plt.show()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAFKCAYAAADMuCxnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl4W/WdL/732bVbiyXbsZ3FWZ3E\n2UgCgYaSsIWy70uHzkzTGdrSDrTcYTq97e/Hc/v70SntzFx6y5QChVIYCsXQNm0pCSk7ZA/Z98WJ\n7Xi3bGs9ks459w8tthPbsR3JR9L5vJ7Hj6Vj6eh7YkdvfXdG0zQNhBBCCNEVq3cBCCGEEEKBTAgh\nhOQFCmRCCCEkD1AgE0IIIXmAApkQQgjJAxTIhBBCSB7g9Xzxjo5AVs/nclng94ezes58QNdVWOi6\nCgtdV2Ep9Ovyeu3D/qyoasg8z+ldhJyg6yosdF2Fha6rsBTrdQFFFsiEEEJIoaJAJoQQQvIABTIh\nhBCSByiQCSGEkDxAgUwIIYTkAQpkQgghJA9QIBNCCCF5gAKZEEIIyQMUyIQQQkgeoEAmhBBC8oCu\na1lnUzASx6aPjiMYlCEJHASehcizEAUOIs9C4LnU/dRtgYXIc+A5BgzD6F18QgghBlc0gfzxnhb8\n9r1jY34eA0BIhXMmrFPBLfKpYE+FeibYhf7b0lkBL6R+1n++/tsiz4FlKfwJIYScq2gC2SQmFxxf\ntaQS0yc5EEuoiMdVxBIKYnEV8UT/7VhCSd6PK4gl1ORjU/cjcgJ9oeTjFFXLejk5lhk+7NO3z6rh\nOx1mxGPxAY8d/FxBYCFlPgwkfyYJLHiOpdo/IYQUiFEF8hNPPIEdO3YgkUjggQcewLvvvov9+/fD\n6XQCANauXYsrrrgC69atw4svvgiWZXHXXXfhzjvvzGnhB3LaJABAaYkJl86vyMo5FVVNBbiKeCq8\n4wkVclwZJuDPvZ38MJAK/szz+j8M9IVimdvZxgCZ4D63Cf+s5nxh6Fr92TX8ZNj3n6//QwQLjqUh\nCYQQMl7nDeTNmzfj6NGjeO211+D3+3Hrrbfikksuwbe//W2sWrUq87hwOIynnnoK9fX1EAQBd9xx\nB66++upMaOea0y4CAHoCsaydk2NZmCUWZilrpxyWpmlIKOmwTgV7XIWcUGC1mtDeGRwQ6MMH/MCa\nf3xAC0AsriAaU9AXjiMWV3Ja+083+5/dAtD/wSAZ8CUOMxKxxAjN/dwIHyKo9k8IKS7nDeRly5Zh\nwYIFAACHw4FIJAJFUc553O7du1FXVwe7Pbn58pIlS7Bz506sXr06y0UeWrqG3BOUJ+T1so1hGAg8\nB4HnYDUN/pnXa0dHSXY/FaiqNmwN/9zmfgXyoJr+gNuDPhgo/S0KCQWBcByxRBTxuIrsxz8G1foH\n1+QHtwoMagE4+2dnNfenPwjwHIPxxn2CYdFdwBuoD4euKzs4loXDKkLgqUWJDHbeQOY4DhaLBQBQ\nX1+Pyy+/HBzH4eWXX8YLL7wAj8eD73//++js7ITb7c48z+12o6OjI3clP4vDIoJlAH+BBvJEY1kG\nJpGHScz9ayVr/9rgJvy4CqvdhPaOwDk1/EyXQPzcrgB5QKtAfMAHCjme/AAQTyhIKLmIf0Kyy2YW\nUGIT4bSKcNoklNgklNhEuFLfS2wSnFYRosDpXVQyQUY9qGvjxo2or6/H888/j3379sHpdKK2thbP\nPPMMfvazn2Hx4sWDHq9p539TdLks4Pns/bE57SYEwnF4vfasnTNfFOM1AcCsya6sn1NRtWTAx5NB\nHYsrkGOp8E4dSx8ffF9NPU5BQsl+nz4hABCLq/AHoujui8LfF0VzR2jEx1vNAtwOCS67Ce4SE9x2\nE1wOE9wOCW6HCW5H8r5ZOvftvFjfN4r1ukYVyB999BGefvppPPfcc7Db7VixYkXmZ6tXr8Zjjz2G\na6+9Fp2dnZnj7e3tWLRo0Yjn9We5mchdYkLDmT60t/cVVf+i12tHR0dA72Jk3URcFwNAYgBJ4gCJ\nAyDk9PUA+n0VGr2vS44r6A3K6AnG0BuKoScgoyckozcYyxzv7o2isS044nlMIpepVTvtEiq8Nogs\nA2e6tm0TUWKVYJa4gn5/1Pv3daFG+jBx3kAOBAJ44okn8Ktf/SozQOub3/wmHn30UVRXV2PLli2Y\nOXMmFi5ciO9973vo6+sDx3HYuXMnvvvd72bvKkbB4zDhWGMPQtEEbObcv/ESQsiFkgQOPpcFPpdl\nxMfFEwp6gzH0hPqDuicop47J6AnE0BuS0dadrui0Dft6M6pKUFfjwYLpHpS5zAUd0MXkvIH81ltv\nwe/34+GHH84cu+222/Dwww/DbDbDYrHghz/8IUwmEx555BGsXbsWDMPgwQcfzAzwmihuR3I0VE9Q\npkAmhBQVgedQ6jSj1Gke8XEJJTmdkhF4NDT60ZOqdfeGkiHe2RvF/pPd2H+yG6/+9Sh8TjPqpntQ\nV+PBnMlO6rPWEaONprM3R7Ld7PDXXWfw328fwrfvXoj50zxZPbeeCr2JZjh0XYWFrquwjHRd/oCM\nfSe6sOdEF/af7EY0lpw5I/Asaqe4UFfjQd10D3znCX89FPrv64KarAtJpoacxbnIhBBSbFx2CSsX\nTsLKhZOQUFQca+rFnhNd2HuiC3uOJ7/wDlDutmBBqvY8q9pJU7VyrDgDmaY+EULIqPAcizlTXJgz\nxYW7Vs1AV28Ue1PhfKDBjw3bGrFhWyMkgUPtFFcmoD0lpvOfnIxJUQVy+g+EApkQQsbHU2LCFYsr\nccXiSsQTKo409WBvqta861gndh1LzqapLLWibroHC2o8mFFVAp6j2vOFKqpA7q8hU5M1IYRcKIFn\nMW+qG/OmunHPlTPR3hPB3uPJ2vOhU368veU03t5yGiaRw7yp7szgMJd9AtYbLkJFFcgOqwiOZaiG\nTAghOeBzmnHlRVW48qIqxOIKDjf2YM/xLuw93oUdRzqw40hydcZqnw0LZ3iwZvlkWEw042W0iiqQ\nGYaB0yZRIBNCSI6JApccjV3jAa4G2rrDyQFhJ7pw+LQfje1BHG3sxSP3LKLm7FEqqkAGkrs+NbQE\noGoaWJrsTgghE6LMbcHVbguuXlYNOabguT8fwI7DHXjxL4fw5etrafGRUSi6jy1OmwRF1RAIx/Uu\nCiGEGJIkcvjKDXMxrcKOT/a14s+bTuldpIJQlIEMAD0BarYmhBC9SAKHf7p9ATwOCW9+eAJbDw69\nlCfpV4SBnNxPkPqRCSFEXyU2CQ/dsRAmkcNzfzqI4829ehcprxVhIKdqyBTIhBCiuyqfDV+7ZT4U\nVcVP39iDjp6I3kXKW8UXyPZ0INNcZEIIyQd1NR588epZCITjeLJ+D8LRhN5FykvFF8hUQyaEkLyz\nekkVrl5ajTOdIfz893uRUFS9i5R3ii6QXTSoixBC8tLdq2dg4XQP9jf48co7R6DjZoN5qegC2Sxx\nEAWWmqwJISTPsCyDB26eh8k+G97fdQYbtjXqXaS8UnSBTKt1EUJI/jKJPP7pjgVw2kT89t1j2Jla\nbpMUYSADyX7kvlAMikp9FIQQkm/cDhMeumMhBIHFM3/cj4bWPr2LlBeKNJBFaAB6qdmaEELy0pRy\nOx64cR7icRVP1u9Bd19U7yLprkgDmaY+EUJIvls8y4u7Vs9AbzCGJ+v3ICIbezpUkQcy9SMTQkg+\nu2ZZNa5YXInG9iB+sW4/VNW4I6+LM5DttHwmIYQUAoZhcN9VMzFvmht7jnfh1XeP6l0k3RRlILuo\nhkwIIQWD51h87eb5qCy1YuP2Jvx1R5PeRdJFUQZy/45P1IdMCCGFwGLi8dAdC+CwCHhl4xHsOd6p\nd5EmXFEGcgnt+EQIIQWn1GnGN29fAJ5j8fM/7Edje1DvIk2oogxkk8jDLHEUyIQQUmCmV5Zg7fW1\nkGMKnqzfbaj38aIMZACp1bqoyZoQQgrN8toy3HZ5Dbr7ZPzh45N6F2fCFHUgByNxxBO0WhchhBSa\nS+eXAwDkuKJzSSZOUQcyAPQaqLmDEEKKRTSWDGKTwOlckolTvIGcmYtMzdaEEFJo0jVjkQK58KVr\nyH6qIRNCSMGR0zVkkQK54GUWBwlQIBNCSKGJpmrIEgVy4aP1rAkhpHCla8gSNVkXPictDkIIIQUr\n3YdspEDm9S5ArpTQFoykwMQi7ehr+xSC5IJoqYBoqQAn2PUuFiG6MGIfctEGssCzsJkFqiGTgpGQ\nuxD27xl0jOVtqXAuh2ielAlphmF0KiUhEyNKNeTi4rSJ6OqL6l0MQkbF4qxF5fxvIxZuQSzSkvwe\nbkG07yiiff1b0rG8BaK5IlOLFs0V4MQSCmlSVGIGHNRV3IFsl9DUEUI0loBJLOpLJUWCE2wwl8yE\nuWRm5pgSD/UHdKQ1GdKB44gGjmcew3LmVDiXQ7RMgmApBy+6KKRJwYoacFBXUadU/2pdMZjcRX2p\npIhxghVmYQbMjhmZY0oigvjZNenACUQDJzKPYTipvyad+s5LbgppUhAyo6yphlwcBk59KnNbdC4N\nIdnD8WZwjhqYHDWZY2oimgro1kxQy8EGyMGGzGMYVkz1R/c3efOSBwxTtBMuSIFKj7I20tKZRR3I\nrtTUJ1qtixgBy5tgsk+DyT4tc0xV5EwzdyzcgnikFXKwEXLwdOYxDCtANJdDSNWkzSWzwPFmPS6B\nkAyZ+pCLS6aGHKCpT8SYWE6CyTYFJtuUzDFViSEeaRvU3C2HmiCHGlPPMcNZeTWs7oXUvE10k+5D\nNtJa1sUdyHZarYuQs7GcCMlWDclWnTmmqnHEI22IBk6gr+0TdJ9eh1DXLriqvwDR7NOxtMSo5LgC\nUWDBGuhDYXEHMi2fSciosKwAyVoFyVoFq3sh/E3rEek9hNZDz8BRtgKO8sv1LiIxGDmmGKr/GCjy\nQHZYBTCgDSYIGQteLIG35i5Eeo+gu+kv6Gv7BCH/fpjY2wBU6V08YhByXDFU/zEwykB+4oknsGPH\nDiQSCTzwwAOoq6vDo48+CkVR4PV68eMf/xiiKGLdunV48cUXwbIs7rrrLtx55525Lv+IOJaFwyrS\n8pmEjIO5ZBYqbFPR1/oh+to349hnz8NcMgeuqjXgRYfexSNFLhpT4HFIehdjQp03kDdv3oyjR4/i\ntddeg9/vx6233ooVK1bgvvvuw3XXXYf/+I//QH19PW655RY89dRTqK+vhyAIuOOOO3D11VfD6XRO\nxHUMy2mT0NIVgqZpNECFkDFiORHOyqtgcS9AoPVthHoOIRo4gZKKK2D3LqfpUiQnNE1DzIA15PP+\nb1q2bBmefPJJAIDD4UAkEsGWLVtw5ZVXAgBWrVqFTZs2Yffu3airq4PdbofJZMKSJUuwc+fO3JZ+\nFJw2EbGEioic0LsohBQs0ezD7GVfg3vyjWAYDj3NG9B6+DnIoSa9i0aKUELRoKia4fqQzxvIHMfB\nYkkuqlFfX4/LL78ckUgEopic4+vxeNDR0YHOzk643e7M89xuNzo6OnJU7NFLj7T2U7M1IReEYVjY\nPItRMfdBWN2LEI+0ou3I8+hufAtqgtaMJ9mTnoNspClPwBgGdW3cuBH19fV4/vnncc0112SOa5o2\n5OOHOz6Qy2UBz2f3H9zrHbxdXWWZA8AZgGPP+VkhKeSyj4Suq7Akr8uO8oovItC9AqcPvoFg53ZE\n+w6hevZNcJUvKsiuoeL+fRUezR8GAJQ4TENeQ6Fe1/mMKpA/+ugjPP3003juuedgt9thsVgQjUZh\nMpnQ1tYGn88Hn8+Hzs7OzHPa29uxaNGiEc/rT/2jZ4vXa0dHR2DQMTHVBtDQ1INKV2GuPjTUdRUD\nuq7Ccu51eeGd8Q/oa9+EvtYPcXLvKzhzchPc1V+AYPLoVs6xMs7vq3Cc6Qwlb6jaOddQyNcFjPxh\n4rxN1oFAAE888QR+8YtfZAZoXXrppVi/fj0AYMOGDVi5ciUWLlyIvXv3oq+vD6FQCDt37sTSpUuz\ndAnj50wtn0lzkQnJPoblUFL+OVTUfg0mxwzIwZNoOfQ0Ah3b9C4aKWDpjSWM1od83hryW2+9Bb/f\nj4cffjhz7N/+7d/wve99D6+99homTZqEW265BYIg4JFHHsHatWvBMAwefPBB2O36Nyv0Lw5CfciE\n5AovueCtuReR3kPwN72NcM9B2L3L9C4WKVD9fcjGGsV/3kC+++67cffdd59z/IUXXjjn2Jo1a7Bm\nzZrslCxLaLUuQiYGwzCwOGthLpkDQNW7OKSAZWrIBtvHvuiv1mYRwLEMrdZFyARJDuoyVlMjya5o\nPDlNleYhFxmWYVBiE6mGTAghBSIWT7awSAZrsjbE1TptEnqCMaijmIpFCCFEX+mtFyWh6BtxBzFM\nICuqhmAkrndRCCGEnIccSzZZm6jJuvhkpj5RPzIhhOS9aDxdQ6ZALjo09YkQQgpHLJbqQ6YacvFx\n2WnqEyGEFAoaZV3EaC4yIYQUDjlGTdZFq3/5TGqyJoSQfCenpj3R0plFKL0FIw3qImOhahpiiopo\n6ksecDuqKANuD/5Z4hAQkuOIqxokjoWJY2HiWZg4Duah7qdumzg2c19kWbAFuGsSIdkgxxJgAAgG\nm4dsiEC2SDwEnqUmawNRNG1QSMrpEE2cHaDKEGGb/IopKsYzc93MsxCZZLDKioq+eAJxdWxnYoBk\ncHMsTPzgsB4c3slQl4a4T4FOClU0rkAUOcP9DRsikBmGgZNW6yoqCVVDe0TGmXDyqzUSQyiuQE4F\nbGyMAQgMDkG3yKdqt8nwk3i2PyBTgZf+2cAvkWNR5nOcsz1cQtX6a9UJFZGBNeyR7idUdEfjkNWx\nrw0tsckAN58V7Mkw5zI/k866n74ujjXWmyHJH3JcNVz/MWCQQAaSA7uONfdCUVVwrLGaQQqdrKho\nDfeHb0tYRltEhjIgcxkA5lTglAr8gNBMf3GZpmJpmGAVWSa1DnP28SwDG8vDJozv+Zka/znhrSAy\nsGY/8H4i+bieWALyOGr7Asv0BzTPwtEgglXUs8K8v9ndPODf2MSxEOj/GRknOZYwXP8xYLBA1jSg\nLxTPTIMi+ScUV3AmHEXLgADuisYHhQnPMKiwSKiwSJhkMWGSRUK5RSzqAOAYBhaeg4XnAIw91dP9\n4cOGd6o5P5K6HUkkm/EjiopgIoHOqAo1GB3Ta/IMMyigBwb22eE9+H4y5IUcfkAi+U2OK7BbRL2L\nMeEMFchAcuoTBbL+NE1DVySGA/5gptZ7JiyjN7VkXpqJYzHVbsYki4RJqRD2mkVw9EY9JizDJJus\n+fHVOjRNg8NtRVNb3znh3d/MfnaYJ38WSajwywkoY1xLnmVwzmA3GhhX/DRNQzSmUJN1MXPa01Of\nirsfWdM0aIk4NDkGVZahyjK0mHzObU2WocoxqDEZDMuCL3GCc5aAd7rAl5SAszvAZKnGqWoaOqPx\nTOi2hKM4E5YRTgzuF7ULHGaXWFI132Tt1yXxVEvKA0wq0EtEHiXjfNuIq2omoIcK8pFCPjDOgXED\nB7tlwjp1v9ZpxXSHZVzXQnInoajQNOMtCgIYKZDzaPlMTVWhxeP9wTgoJJP304GpxWSEOQ1Bf2Bw\nsKbCdOA50s9HNna1YllwDgf4Eif4khLwTie4Eid4pzN5LH3f4QDD9f/HSagq2iKxQU3OrWH5nEFW\nbklAbakDHo5LBrBVgt1gO7sYjcAm+5Xt4+xHP3tgXFQZPPhtrAPjTgcj+PrcyVm4MpJN6Z2eqA+5\niGUCOUtzkVVZRnDHdijB4LDBqMZiQ4SsDC2WpQ8FHAdWFMFIEliTCayjJHlbksCKEhhJHHA7eZxJ\n3U/eFsGKEjRFQaKnB0pvDxK9vUj09mTux1rOQD7VMOTLxwUR3Z4y+CunwF9WiU6XF36LHeqAmjUL\nwGcWM7Xe9HcTz8HrtZ8zGpmQ4WRrYNy6U+3Y0x3EklJHdgtIsiK9SpdIgVy8sr2edd/mT9H+0ovn\nfRzD85kw5Kw28G5PMgxFCWwqMAeH5FmBKUlw+VzoiygDnpf6OZ/7X5+maVAjYSR6etHS3YMjQRkt\nCQ1tjIAeQQIGNCdz8TjcHS3wdLbC09mKKbOmY84N1xf1YCtSODiGQTShYr8/CLckYFlpid5FIkOQ\nUzs9GW3rRcBAgVxize7ymZqcDHb3F26Aefac/pAcFKzioObc8Srx2hHTqSbJMAw4ixWcxYr/bosi\nxAkAlxxsVZPu67Uma75uKNB63Yh3laL5P9+GhYlDYG/UpdyEDGVjcxcUDbi60kPzrPNUZutFCuTi\nZZZ4mEQu64O6TDXTYZ03P6vnzFdfnFGBUELBJIsEpzjMYCuzBWJ5OXiPB3Jz48QXkpBhtIRl7O4O\nYJJFQp3bpndxyDBkA/chG6ot0WmTin6UdS5NtZsxz2WDSxLOO/JZqqyC0teHRKBvgkpHyMg2NHVC\nA3BNlYemQ+WxdJO1EfuQDRbIIgLhOBLK2JchJGMjVVUDAGJNTTqXhBDgZCCCw71hTLObMZOmOuW1\nTA3ZgE3WxgrkLA/sIsMTK6sAAHIzBTLRl6ZpWN/UCQBYU1VK89rznJH7kI0VyHk0F7nYSVWpQKYa\nMtHZoZ4QTgejmOu0otpm0rs45DxiqRqyEVfqMmYg077IOSeWlQMchxjVkImOVE3D+uYuMACuqSrV\nuzhkFKiGbBBOmzGWz8wHDM9DLK+AfKYZ2ji2DiQkG3Z1BdAeiWFJqQM+s/E2KyhENMraIHLSZK1R\n2AxHqqqCJsuId3bqXRRiQAlVxcbmLvAMgysnufUuDhmlTA2ZArm4ZXNQl1BWDgDo2/TpBZ+rWEmp\ngV0xmo9MdLClvRc9sQQu8ZXAKY1zvU0y4TJ9yNRkXdxcWWyyti5YCNP0GQju3IHwoYMXfL5iJNLA\nLqKTqKLgvRY/JI7FFVQ7LijUh2wQAs/BauKz0mTNMAx899wHAOh47TfUTzoEqTI5F5mmPpGJ9nFr\nD8IJBSvLXbCMcw9oog+ZRlkbh9MuZW2UtWlaDRwrLoPceBp9n3yUlXMWE97tBms20+IgZEIF4wl8\n3OqHjedwWZlT7+KQMZLjChgAIm+4eDJgINskhOVEZnm2C+W57Q4woojO370BNRrJyjmLBcMwkKqq\nEWtrhZqtLScJOY/3zvgRUzWsmuSGxBnuLa7gyTEFksgZcgEXw/21pqc+9WZp6pPgcsF93fVQ+vrQ\n/dafs3LOYiJWVgGahljLGb2LQgygW45ja0dPcntFL22vWIiiccWQ/ceAIQM5+1OfXNesAe9yw7/h\nbcQ7OrJ23mKQHmlNA7vIRPhranvFqyrd4Gl7xYIkxxVD9h8DBg5kfxZX62IlCaV33AktkUDHG7/N\n2nmLQXoJTVqxi+RaS1jGrq4AKswiFrjteheHjJMcUwy5KAhg4EDO9mpd9uWXwFQzHcHt2xA+cjir\n5y5kYmUlABppTXKrW47jN8dbUtsrltL2igVK07RMH7IRGS+Q7blZPpNhGHjvvhcATYMaiLNYwbvd\n1GRNcqYpFMXPDzSiMxrH5eUuzCqh7RULVTyhQoMxpzwBBgxkVw53fDJPnwH7xZdAPtVAK3gNIFVW\nQentgRIM6l0UUmQO9QTx7KEmhBMKbpzsxZpq2l6xkBl5URDAgIHssKZqyDna8an09juT06DerIca\njebkNQpNZm/kJlpCk2TPlvYevHS0BQDwxRkVWEFzjguekTeWAAwYyDzHwmERcrbjk+D2wHXtdVB6\ne9D9Nk2DAgbsjUz9yCQLVE3D+sZO/OFUByw8h6/MrsJcl03vYpEsSAeySDVk43DaJfQEY9A0LSfn\nd6/5AjinE/71byPeRTsdpZfQpJHW5EIlVBWvn2jDB61+eCQBX62tQrXNpHexSJakF2yiGrKBOG0S\n5LiCaCw7q3WdjZUkeG+7E1o8js43Xs/JaxQSsaIC4Dga2EUuSCSh4IUjZ7C7O4DJVhO+WlsNj4n2\nOC4m1IdsQLma+jSQ/ZIVkKZOQ2DrFkSOHc3Z6xQChuchlldAbm6m0edkXHrkOH5xsAknAxHMc1mx\ndk4lrAatRRUz6kM2oPTymbka2AUADMvCdzftBpUmVVZBk6NIdHXpXRRSYM6EZfz8YCPaozFcWubE\nvdMrILCGfOsqeukma+pDHsGRI0dw1VVX4eWXXwYAfOc738GNN96I+++/H/fffz/ef/99AMC6detw\n++23484778Trr+dvU63TnrupTwOZZ86EfdlyRE+eQGDL5py+Vr7LDOyikdZkDI70hvDMwUYE4wqu\nry7FDZO9tOhHETN6DZk/3wPC4TB+8IMfYMWKFYOOf/vb38aqVasGPe6pp55CfX09BEHAHXfcgauv\nvhpOZ/5NRZiIJuu00jvuQvCzneh883XYllwEVpJy/pr5KDP1qbkJtsVLdC4NKQTbO3rx+4Z2sAyD\ne6eXYz4th1n00uN6qA95GKIo4tlnn4XP5xvxcbt370ZdXR3sdjtMJhOWLFmCnTt3Zq2g2eTKwXrW\nwxE8pXBdswYJvx/db7+V89fLV/01ZBrYRUamaRo2NnfhzYZ2SByLtbMrKYwNIpYe1EU15GEewPPg\n+XMf9vLLL+OFF16Ax+PB97//fXR2dsLtdmd+7na70XGenY9cLgt4Prv/8F7v+f/j8pIAAIjE1VE9\n/kK57r8HOzd9jJ71f0HNLddDKvWM+RwTUc5c0kptOG2xQGk9M+haCv26hkPXNT4JVcWv957GpjPd\nKDWLeGjZDJRPwLQm+n3lBzaVB+U++4hlL7TrGq3zBvJQbr75ZjidTtTW1uKZZ57Bz372MyxevHjQ\nY0Yzx9fvD4/n5Yfl9drR0RE47+NUVQPLMGjrCo3q8dngvvl2tP3qlzj87Auo+MoDY3ruaK8r3wkV\nkxA5eQJtZ7rBCoLu13XySAcURYPFKsJiE2G1SRCy0FSm93XlSq6vK6ooeOVYC471RVBllXD/zEng\nInF0ROLjOp+iqJAjcUQicciRBKKROORoApMmO1HiMmceR7+v/OHvjQAAIiF52LIX4nUNNNKHiXEF\n8sD+5NWrV+Oxxx7Dtddei87O/kUw2tvbsWjRovGcPudYlkGJTZyQPuQ0x6WXoefdjQhs3gTn6qtg\nrpk+Ya+dL6SqKkSPH0Os5QxiysrUAAAgAElEQVRMk6foWpZQQMbbb+4/57ggcslwtoqw2KXkd5sI\ni02C1Za6bZUgShytmZxFvbEEXjzSjNZIDHOcVtxTUw6RS/aoaZqGRFxBNBWqg78SkIc4Fo3EER9m\nnYEZc324+qa5E3l5ZJQyfcjUZD163/zmN/Hoo4+iuroaW7ZswcyZM7Fw4UJ873vfQ19fHziOw86d\nO/Hd73432+XNGqdNRGN7EJqmTcgbK8Oy8N5zH5qe+CE6Xn0F1f/6PcO9oUuV/Xsj6x3IFpuIm+5d\niO7OEMLBGMLBGEJBOfk9FENvd2TE5/MCm6pZ9we11SbBYhURqIoinlBgsYmQTLzhfs8jUVUNMTkV\nrOFkgLaEotgYiyDCANVRDb593fjL1jZEo/0BrCqjW1WP41mYzAIcThNMZgEmswDJLMBk5mEyJe9X\n17jPfyKii5jBFwY5byDv27cPP/rRj9Dc3Aye57F+/Xr8zd/8DR5++GGYzWZYLBb88Ic/hMlkwiOP\nPIK1a9eCYRg8+OCDsNvzt53faZNwsiWAUDQBm1mYkNe0zJoN20VLEdyxHYGtW+C4+JIJed18IVYl\nl9DMh4FdDMOgcooLlVNcQ/5cUVREQjGEzgrrcGjA7WAMbc29GKl3huMYWGxSqmadCm2bODjEbSJM\nZqHggltJqEPWWKOR+IBa6+BarRxNZJ6vAYi6JXTVuaHxLEqO9kI7HcTJ1M9FiYfJzKPUZxsQrnzm\ndv8XnwlewaA1q2KRXqnLRIE8tPnz5+Oll1465/i11157zrE1a9ZgzZo12SlZjmXmIgfkCQtkIDkN\nKrR7Fzrf+C1sixYbahqUVFk4m0xwHAubwwSbY+QBRaqqIhKKDwpqqEBHeyAV5DGEQzLaz/SNGNws\ny8Bs7Q/qQTVva3+ImywiWDa7wa1pGuIxZchg7Q/XBFRFRV9vNHM8ER/dYjcsy0Ay87DYRJR4rYi4\nJPTaOHRLLCJscqrHFWYr5q/0DqrRsrT4h+HIMQUMk9wEyIjG1WRdDAbORa7yTdxOMaLXB+dV18D/\n9lvwv7MenhtumrDX1htntYJ3uYpqkwmWZWG1S7DaJXiRbBEaatCJpmmIhOMIB+VMrfvs2nYoKKOz\nPQi1ZfjkZhjAbB1c2+6vcSdD3GwRkUgM6HMNxxGN9gfr2bVaOZKAqo6uSZgXkk3CTrclUzuVzEKy\nOdgyuMZqMgsQJR69moojfWEc7gnhVDCCdOuziWNR57DgsnInJtvMI78wMQQ5rsAkGnd8hoEDObl8\npn8CB3alua+/EX2ffIzut/6Eks+tBO8cutm0GImV1Qjv2wMlFAKKdOrCUBiGSfY5W0WUlg3/OE3T\nIEcT/f3ZwRjCA2+Hkrf9XWF0tgUvqEySKRmcjhJzf7AO0RScrrVWV7vg7zn/zAhZUXG8L4wjPX04\n0htGT6y/mXqSRcKsEgtml1hRZTOBM+gbLxmaHFMMO6ALMHAgu2wTs3zmUDizGaW33o62X7+Azjfr\nUf7lf5jwMuhFqqpCeN+eZLP11HK9i5N3GIbJhKDHO/zjNC05OGpgaIdCyZp2JBwDz3PDBqvJzEMy\nCWNu+uaHeaPUNA3t0RiO9IZxpDeEhkAUSqp93sSxqHPZMNtpxcwSC+yCYd9yyChE4wrMknH/Rgx7\n5RO5fOZQHJ9biZ73NqLv00/gXH0VTFOn6VKOiZYZad3UCGCpvoUpYAzDQDIJkEwCXKXWCX99WVFx\noi+Mw70hqgWTrJHjSqb10oiMG8gDBnXpgWFZeO++D00/+RHaX30F1f/yXUP0m2SW0CyifmQj0DQN\nZwIRbG71D1sLnuW0YhbVgsk4qZqGWEwx7MYSgIED2WriwXOMbjVkALDMqYV18RKEPtuJ4PZtsC9b\nrltZJopQXgGwbF5MfSIj668FJ5uiqRZMcikeV6EBkETDxpJxA5lhGDhtki59yAN577gboT270VH/\nGqyLFoEViru5hhUEiOXliDU3jWp5VTJxNE1DRzSeaoY+txa8tMKJKSaJasEkJ+TMxhLGnPIEGDiQ\ngWQ/8okzfcm1rbM8t3O0xLIyuK66Gv71b6Nn40a4r/uCLuWYSFJlFWJnzkBu7wBYmu6ip7HUgst9\njoJeQ5jkt6jBV+kCDB/IIlRNQyAcQ4lNvwU63NffhODOHVACfbqVYSKJlVXAtq0InzoFTJujd3EM\nJV0LPpKqBZ+kvmCSJ+TUOtYmA//dGffKMXCktb6BzFksmPr4E4YY1AUAUmoJzdCp0zBRIOcc9QWT\nQpAOZFGkJmtDcqVGWvuDMqZA30UqjBLGQP/Up/CpU8j9TrfGoGgaQnEFwXgCgdT33riChkCYasGk\nIKT7kGmUtUHpPRfZqHiPB4xkQvjUadC+O8NTNQ3hhIJgXEEgnkAwPuB2YmD4KggnFAw3RC5dC55V\nYkU11YJJnspsvUijrI0pPQFdr7nI2SKfOYPYmWZwNlvqyw7OZgPD59+vV43HkOjqBu9yItJ8Bloi\nkZflzBVN0xBV1EwtNv09GFcQSAwO3VBcwfm2bzBxLGwCB59ZhE3gYON52AUOdoGDTeBRYZHgMPAb\nHCkccjzZlWLUnZ4AoweyXb/lM7Op9dmnITeePuc4azKBtdnAWW3nhDWXPm63g7VaMz9jxQubdqXK\nMuJdnUh0dSHe2Zm6nfwe7+qC0ts76PFKMAje6byg18wnMUXFzlY/mjqDyXBNnBW6cSXTfDwcgWVg\nF3hU2YRMsNoHhK1N4JPhK3AQaEckUiTk1O5hIk17MqZiabIu//JXED58GEowACUUhBIIQg0Fk/eD\nIcTONEOLx0d1LkYU+8PaZgdns4JNh7jVBs6e/K4pSn/YdibDNtHVCSUwzLQYjoPgdkOcUwvBUwqh\ntBS+BbVIlJRk8V9Cfx+0+PFeS/c5x3mGgU3gUGERYc8EajpouUHHJINuPUeMjUZZGzyQTSIHSeAK\nPpCl6smQqieP+BhVlqEEg6nATga3Ggwmj6W/Bvws1t4GbYha93AYngfv8UCqngze48mEruApBe8p\nBe90gjmrNuceYpvCQrfc50CF2wo1Eh8UuiaONdTAPULGKpoa/U/zkA0quVqXWPBN1qPBShJYSYLg\n8Yz6OWo8BiUYSgZ3psadDG+GZcGnAlfwlIJzOM4JXCMqEQXMqHQX3QcNQnItlmqypu0XDcxpk9De\n2IOEooKnpsJBWEEE6xIBl3H2ayaE6INW6gIMn0BOuwQNQF+o+GvJhBCSr+RUk7WR5yFTIKemPvkL\nvB+ZEEIKWXqUNdWQDSwz0jpANWRCCNFLuoZs5D5kwweyy14cU58IIaSQReMKOJYBzxl3NoLhA7lY\n5iITQkghk2MKRIEz9PRACuT08pkUyIQQohs5rhh62UyAAjmz7aIR5iITQki+kmOKofuPAQpkSAIH\ni8RTDZkQQnQUjSuGHmENUCADSM5FLvQdnwghpFCpmoZYXKUast4FyAdOm4hQNIF4QtG7KIQQYjix\n1Cpd1IdMMiOt/dSPTAghEy690xPVkMmAxUGo2ZoQQiaaHKdABiiQAdDUJ0II0VM0RhtLABTIAAYu\nDkJN1oQQMtFk6kMGQIEMIDnKGqAaMiGE6CEdyCI1WRMXLZ9JCCG6SQ/qMvLWiwAFMgCgJN2HTIO6\nCCFkwlEfchIFMgCeY2G3CNSHTAghOqA+5CQK5BSnTaIma0II0QH1ISdRIKc4bRKiMQUROaF3UQgh\nxFCoDzmJAjklPRe5N0TN1oQQMpGoDzmJAjmFVusihBB9xGilLgAUyBnpuch+6kcmhJAJFaVBXQAo\nkDNo+UxCCNGHTE3WACiQM/qbrKkPmRBCJhJtLpFEgZzipNW6CCFEF3JMAccy4DljR9Korv7IkSO4\n6qqr8PLLLwMAWlpacP/99+O+++7DQw89hFgsWatct24dbr/9dtx55514/fXXc1fqHCiximAYCmRC\nCJlo0bhi+P5jYBSBHA6H8YMf/AArVqzIHPvpT3+K++67D6+88gqmTJmC+vp6hMNhPPXUU/jVr36F\nl156CS+++CJ6enpyWvhsYlkGJVaRApkQQiaYHFMMvygIMIpAFkURzz77LHw+X+bYli1bcOWVVwIA\nVq1ahU2bNmH37t2oq6uD3W6HyWTCkiVLsHPnztyVPAeSq3XFoGma3kUhhBDDkKmGDGAUgczzPEwm\n06BjkUgEopgclezxeNDR0YHOzk643e7MY9xuNzo6OrJc3Nxy2iTEEyrCtFoXIYRMGDmmGH5AFwDw\nF3qC4WqTo6llulwW8Hx2fwler33cz63w2rDrWCcYgb+g8+RCvpUnW+i6CgtdV2EphOtSVA2xhAq7\nVRp1eQvhusZjXIFssVgQjUZhMpnQ1tYGn88Hn8+Hzs7OzGPa29uxaNGiEc/j94fH8/LD8nrt6OgI\njPv5Es8AAE6e9sPCMdkq1gW70OvKV3RdhYWuq7AUynWl9w9goI2qvIVyXcMZ6cPEuMaYX3rppVi/\nfj0AYMOGDVi5ciUWLlyIvXv3oq+vD6FQCDt37sTSpUvHV2Kd0NQnQgiZWLT1Yr/z1pD37duHH/3o\nR2hubgbP81i/fj1+8pOf4Dvf+Q5ee+01TJo0CbfccgsEQcAjjzyCtWvXgmEYPPjgg7DbC6tZIR3I\nXX1RnUtCCCHGkFmli/qQzx/I8+fPx0svvXTO8RdeeOGcY2vWrMGaNWuyUzIdTCm3g2GAfSe6cdNl\n0/QuDiGEFD1apaufsZdFOUuJVcScyS4ca+5FZ29E7+IQQkjRo60X+1Egn2V5bXK+9bZD7TqXhBBC\nih/1IfejQD7LRbN94FgGWw9QIBNCSK6l+5BppS4K5HPYzALmTXPjVFsArd3ZnZZFCCFksEwNmQKZ\nAnko6WbrrQfbdC4JIYQUN+pD7keBPITFM73gORbbDlKzNSGE5BL1IfejQB6CWeKxcLoHzZ0hNHUE\n9S4OIYQULZqH3I8CeRjL55YBoGZrQgjJpUA4BiBZETI6CuRhLJjugSRw2HqgnbZjJISQHDnVFgDH\nMqjwWPUuiu4okIchCRwWzSxFe08EDa2Fu5A5IYTkq4SiorE9hCqfDQJPcUT/AiOg0daEEJI7zR0h\nJBQV08oLa9+DXKFAHsH8aR6YJR5bD7ZDpWZrQgjJqobWPgDJfQQIBfKIBJ7FRbO88AdkHG/u1bs4\nhBBSVNLdgVPLHTqXJD9QIJ/H8rmpZmtaSpMQQrKqoSUAnmNR6aUBXQAF8nnVTnHBZhaw7VAbFFXV\nuziEEFIU4gkVTR1BVPts4DmKIoAC+bw4lsWyOT70heM4fLpH7+IQQkhRaOoIQlE1TK2g/uM0CuRR\noNHWhBCSXf39xxTIaRTIozCzygmnTcSOwx1IKNRsTQghF6qhJTnCmgZ09aNAHgWWZbBsThlC0QT2\nn+zWuziEEFLwGloDEHgWk0otehclb1Agj1JmtDXtAEUIIRckFlfQ3BHC5DIbOJZiKI3+JUappsKB\n0hITPjvagVhquzBCCCFj19gehKpp1Fx9FgrkUWIYBstryxCNKdh7okvv4hBCSMGiAV1Do0Aeg/Ro\n6y3UbE0IIeOWXjJzagXVkAeiQB6Dap8NFR4L9hzrRERO6F0cQggpSA2tAUgChwo3DegaiAJ5DBiG\nwbI5PsQSKnYf69S7OIQQUnDkmIIznckBXSzL6F2cvEKBPEbLa8sA0GhrQggZj9PtAWgazT8eCgXy\nGE0qtaLaZ8PeE10IReN6F4cQQgpKQ0tqQBctmXkOCuRxWF7rg6Jq2HmkQ++iEEIGiCcU9AZl+AOy\n3kUhw6AR1sPj9S5AIVpeW4Y3PjiBrQfbsXLBJL2LQ0hR0DQNclxBRFYQjsYRlhMIRxMIywlEBtwe\n6lgk9fiEomXON6nUiqWzvVg6x4fKUisYhvor80FDax9MIocyGtB1DgrkcfA6zaiZ5MDBBj/6QjE4\nrKLeRSJEd6qmISoryaCUE4NCtf9YOkBT9wfejiagatr5X2gAnmNgMQmwmAR4SsywmHhYJB5yXMGB\nBj/WfdKAdZ80oNxtwdI5Xiyd7UO1z0bhrJOInEBrVxizJzvB0u/gHBTI47S8tgwnzvRhx+F2rFpS\npXdxCNFFb1DG3hPd2HuiC/tPdiM8xumAosDCIvFwWEWUuc2wSEImVC0mHmap/7ZF4mFO/yx1TOC5\nYc8dkRPYe6IL2w61Y+/xLvzp01P406en4HOZsXS2D0vneDGljJpNJ9LptgA0AFOouXpIFMjjtGyO\nD6/99Si2HKRAJsahqhpOtPRhz/Eu7D3RhVOp/kAA8DgkzKgqgdXEwyIJ/eE5RJimb+dyY3qzxGN5\nbRmW15ZBTq2wt/1wO3Yf68Jbm0/hrc2nUFpiwuWLqzB3shPTKuxUc86x/v5jGmE9FArkcXLZJcys\nduJoYw+6+6JwO0x6F4mQnAiEY9h3shtHmo5g+8FWhKLJWjDHMqid4kJdjQd10z2Y5LHkbaBJIoel\nc3xYOscHOa5g34lu7Djcjs+OdeLN94/hTSQ/UFw0O/mYmkkOalLNgUwg0wjrIVEgX4CLa3040tiD\n7Yfacc3yyXoXh5CsUDUNp1oD2Hu8C3tOdOHkmT6ke3ZddgkXzfairqYUc6e6YJYK7y1EEjhcNNuL\ni2Z7EU8oaOyK4K9bT2PXsQ5s2NaIDdsaM9e5dLYPM6pKKJyzpKE1ALPEw+c0612UvFR4/5vyyEWz\nffjvd45iKwUyKXChaBz7T3Zjz/Eu7DvRhb5wco49yzCYWe1EXY0bn186GVaeydta8HgIPIeL51eg\npsyGeELFgYZubD/cjs+OdGLj9iZs3N6EEpuIi2Z5sWyODzOrnLS61DiFowm0dYdRO8VVVH9D2USB\nfAEcVhG1U13Yf7IbHT0RePP0U19PUMbvPzqJWEKBWUwOlDFLHExi8rtZ5GGSzrotchB4lv7jFClN\n09DYHsSeVC34eHMv0gOcHVYRl9WVY8H0Usyb6oLFJAAAvF47OjoCI5y1sAk8i4UzSrFwRikSa1Qc\nOuXHtkPt2HmkA+/ubMa7O5vhsAhYMtuHZbO9mDXZSXv5jsGpNmquPh8K5Au0vNaH/Se7sfVgG65f\nMVXv4pyjLxTDj3/zGVq6wmN+LscyMIkczBLfH94SD5PIYUq5HasXV0EShx/lSvJLOJrAgYZu7DmR\nHJDVG4wBABgGmD6pBHU1biyYXorqMpvhm2h5jsX8Gg/m13hw/7WzcTjVNbXzSAfe/6wZ73/WDJtZ\nwJJZXiyd48Wcya6cDlArBpkdnmhA17AokC/QRbO8+PXbh7H1YHveBXIwEsdPXk2G8TXLqnHNsmpE\n5AQiseRc0YicQHSo2zEFUTmBSCyRnFcaS6CrL4qonMj0JW492I4N2xpx68oaXFZXTjWFPKVpGj7d\n14qP97TgWHMvFDX5G7SZBayYV4a66R7Mn+aBzSzoXNL8xXMs5k11Y95UN/7mmlk40tiL7YfbseNw\nBz7cfQYf7j4Dq4nH4lnJPue5Uymch5JZMpOmPA2LAvkCWUwC6mo82HWsE2c6Q5hUatW7SACAcDSO\nf39tF5o6Qli9pBJ3r55xwc3PqqZBjikIRxP4YPcZbNh6Gr/6yyFs2NaIO66YjoXTPdTEnUcSioqX\nNxzGh7tbwCDZVFhX48GC6aWYWm6nvtBx4FgWtVNcqJ3iwhevmoWjTT3YfrgDOw634+M9Lfh4Twss\nEo/FM0tR7bPBahZgNQuwmQRYzTyspuQ8ayMGdkNrH6wmHqUlNCNlOBTIWbB8rg+7jnVi68E23LKy\nRu/iICIn8J+/3Y1TrQFcvrAC9109KytByTJMqv+Zx22X12DV4kr84eMT+GhPC35avwezq524a/UM\nTKNNx3UXCMfwX7/bh8ONPZhSZseDt85HaZ6OcShULMtg9mQXZk924d6rZuJ4cy+2H+rA9sPt+GRf\n64jPNYkcrANCOhnafDLATQKsJh62VJhbBxwX+MIM8lA0jo6eKOZNc9OH9hFQIGfBohmlEHkWWw+2\n4+bPTdP1D06OKXjy9d04fqYPK+aV40vXzslZf6DLLuHvrqvF1UurUf/+cew+3oUfvLgdy2t9uO3y\nGvhctFatHpo7Q/hp/W509ESxdLYXa6+fS339OcYyDGZWOTGzyom7r5yB020BdPXKCEXjya9IIvU9\njlA0kfoeR5s/AjkWHPXriAKbCmwBtkyY94e61cRjarkj71bCog0lRocCOQtMIo8FM0qx/VA7GtuD\nmKzTcnyxuIKfvrEHR5p6sWyOD1++fs6ENEtWem146M6FOHTKj9++dwxbDyb711YtqcSNl06F3UJr\nfU+UPce78It1+xCRFdx46VTcvHKa4QdoTTSWYTC13IGp5aN7fEJREYrEERwQ1JkAPyvM04/p6oui\nqWP4ZUrnT3Pjb2+YB481P8YGNLTQgK7RoEDOkotrfdh+qB1bD7brEsjxhIr/+v0+HDzlx+KZpfiH\nG+dO+ECrOVNc+N7fLsX2Q+2of/84Nm5vwid7W/CFS6bg6qXVEAWqpeWKpml4Z3sTXnv3KHiOxQM3\nzcPFc8v0LhYZBZ5jUWKTUGKTxvQ8RVURjiYQHFDr7gvHsGlfK/ad7MY//5+PMHeqCzddNg2zqp05\nKv3opGvI02jK04gokLOkrsYDk8hh68E23P75mglttk4oKp7+wz7sOd6F+TVufPXm+boNGmEZBstr\ny7B4phfvf9aMP37agDc+OIF3dzbj1pU1uHR+OQ0myrKBg7dKbCK+edsC1Eyimkix41gWdot4TgvU\nygWTcPi0H29va8Tuo5040ODHnMlO3HTZNMyZ4tKlrA0tAdgtAlz2sX3oMJpxBfKWLVvw0EMPYebM\nmQCAWbNm4Stf+QoeffRRKIoCr9eLH//4xxBF4zRVigKHxTO92LS/FSda+jB9UsmEvK6iqnj2jwfw\n2dFO1E5x4Ru31uXFwA+BZ3H1smpcVleOtzafxjvbG/H8WwexYdtp3LlqBuZPc+tdxKJw9uCtb95e\nR+uqE8ye7MLnLpqMTz9rxLpPGrD/ZDcOnf4Ms6qduOmyqRO6WlYgHENXXxR1NTQL43y4xx577LGx\nPqm5uRnd3d149tlncdttt+Hzn/88Hn/8cdxwww34zne+g4MHD+L06dOoq6sb8TzhcGy85R6S1Spl\n/ZxjIfAMNh9og0ngUVfjydp5h7suVdPw/J8PYcvBNsysKsHDdy7Mu8E7As9h7lQ3LptfjlA0jgMn\n/di0vw1Hm3oxrbIEpjz48JBtE/V32NwZwo9/8xlOtwexdLYX/3T7gpz21+v9/ytXivm6TDyLS+eX\nY/40N3pDMRxo8OPTfa04cMoPl12C12nOeUgebezBpv1tuLi2DLVZqKEX+u/Lah2+lSBr74ZbtmzB\nlVdeCQBYtWoVNm3alK1TF4y5U92wmnhsPdQGVR3bRutjpWoafv32IWza34qaSY68DOOB3A4T1l4/\nF499eTnm17hx8JQf3/rPD/DMuv3o7InoXbyCs+d4Fx5/aTs6eqK48dKp+Oot8/P690/0Nb0y+YH9\n+3+7FItmlOJYUy/+47XdePylHdhzvAualrv3q5O0w9OojbsP+dixY/jqV7+K3t5efOMb30AkEsk0\nUXs8HnR0dGStkIWC51hcNNuLD3e34GhTD2ZPzk1/jaZp+M07R/Hh7hZMKbPj23ctLJhdd6p9Nnz7\nrkXY39CN3310EpsPtGH74XasXlKFGy6dSitGnQcN3iIXYlqFA/90xwKcag1g3Scn8dnRTvzv13dj\nWoUdN142LSeL+9AI69Eb17v41KlT8Y1vfAPXXXcdGhsb8aUvfQmKomR+PtpPWy6XBTyf3U/1Xq++\nn8KuuWQaPtzdgj0NfnzuouztAJW+Lk3T8Pwf9+OvO5swtcKB//9rl8FhLby++iu8dlx+0WR8uKsZ\nL711ABu2NeKTfa2468qZuOFzNQU/IjsXf4fxhIqn39yDDVtOwe2Q8D///mLMytGHvuHo/f8rV4x2\nXV6vHUvrJuHkmV68+s5hfJpa3GfBjFL8r39cAS6Lg0Ib24NwOyTMqinN2jmL9fc1rkAuKyvDF77w\nBQDA5MmTUVpair179yIajcJkMqGtrQ0+n++85/H7x77hwUjyYTea8hIJDquIjz5rxm2fm5qVqUcD\nr+vND0/gT582oMJjwUN3LIAcltERli/4NfTg9doxr7oEP1i7HO/ubMafPm3AC386gHUfHsetl9fg\nknnlBTmHNhd/h0MN3nKZ+Qn9e8+H/1+5YOTrsgksvvKFWqxZVo1X3jmCPcc68cbGw1i1pCorZegN\nyujsjWLhdE/W/o0L/fc10oeJcaXFunXr8Mtf/hIA0NHRga6uLtx2221Yv349AGDDhg1YuXLleE5d\n8FiWwbLZPgQjcRw85c/quf/4yUn86dMG+Jxm/I97FqOkAGvGQxF4Dtcun4x/++oKrLl4MnpDcTz3\np4P4Xy9sw/6T3XoXT3fNnSH8f7/ejsONPVg624vvfHEJjaTOY6qm4uWDr+O9xo+haqrexRmVKq8N\nD9w0D5LI4XcfnUQ4Gs/KeTMrdNFyuqMyrkBevXo1tm3bhvvuuw9f//rX8dhjj+Fb3/oWfv/73+O+\n++5DT08PbrnllmyXtWAsn5tsHdh6oD1r53x7y2n87qOT8DhM+Od7FxflfD6rScBdq2bg8X+8GCvm\nlaOxPYh/f20X/v21XTjdVrifiC8EDd4qPK2hdmxq2Yb6o+vw890voC9WGH+7JTYJN6yYgmAkjj99\neior5zxFS2aOybiarG02G55++ulzjr/wwgsXXKBiML2yBG6HhB1HOnD/tbMveF7wnz4+gd++dwwu\nu4R/vm8xPEW+W0ppiRn/cONcXLOsGq+/fwz7T3bjwMlurJhfjltX1hT99QOpwVvbGvHae8do8FaB\n6Yh0AgBsghUHug/j8a3/ib+tvQe1nlk6l+z8rllWjfc/O4N3tjfiisWTLng9elrDemyKbxJoHmAZ\nBsvnlCEiJ7DvZNcFnXvN8PYAACAASURBVOuDXc34xe/2wmEV8c/3LobPQDv2TCm343/csxjfvnsh\nKr02fLqvFf/6zGa8/t6xrDWp5aOEouLFtw/h1XePwWEV8S/3LaEwLiDt4WQg3zv7Ntw24waE4xH8\nbPdz+N2xPyOhDr/+dD4QeA53rpoORdXw+nvHL/h8J1v74LKPfVlQo6JAzpFMs/XB8Tdbf7qvBb9+\n+3AyjO9ZhHK3MXdPmj/Ng8f+fhnWXl8Lh1XAX7acxr88vQkbtp5GPKFvH90R/3Gc6mvM2vkC4Rj+\n/dVdmSlt3//SUloGs8Cka8g+ixdXTr4c/2Ppg/CZS7Hx9Af49x3/lQnsfLVsjg8zKkuw40gHDl3A\nOBh/QEZvMEa14zGgQM6RKWV2+Jxm7DraCTmunP8JZ9l6sA2//PNBmCUeP3jgUlR6bTkoZeFgWQaX\n1VXg8X+4BHdeMR2qBrz67jH8z2c3Y/P+Vqg5XNhgJK8cqsdPP3s2K/2ENHirOKQDt9ScXK1vsr0K\n/7LsIVxSvhSnA034t23/G1taduhZxBExDIN7rkwui/zqu0fHvchRQ2tq/nEBD+jqCHfhUPfRCXs9\nCuQcYRgGy+f6IMcV7Dk+tmbrnUc68My6A5AEDo/cswg1lROzLnYhEAUO110yBT/66gpcs6wa/oCM\nZ/54AD/41XYcbJj4EdkekxtRJYo/Hl9/QeehwVvFoyPSBZfkhMj1L3Jj4iXcP/cu/N3ce8GAwa8P\nvoZf7X8V0URUx5IOr2aSAyvmleF0WxCf7GsZ1zkaWgq7//hEbwN+tP1JPLv31xM2Wp4COYeW1yb7\n/bYeaBv1c/Yc78TPf78PAs/iW3ctxLQC/nSZSzazgHuunInH//ESXDy3DKfaAvjxq7vwn7/djab2\n0W/4fqGq7ZUAgE0t23A60DTm52uahg1bT+PJ+t1IKBoeuGkebr28piDnXxMgpsTQI/fCaxl6EYxl\n5Yvxr8sfxhRHNba17cQPtz2Z1S6PbLr989Mh8ize/OAEorGx932fSs2MmFKAgXyw+wj+z2fPQlZi\nuGf2bWCZiYlKCuQcqvLaUFlqxe7jXYjI5/+DPtDQjZ+9uQ8sy+ChOxZgZpW+e5gWAq/TjAdumof/\n5++WYs5kJ/ae6ML/+/xWPP/ng+juy33tIx3IGjTUH1k3pjWBafBW8emIJFvDfObhN5cpNXvwyJKv\n45opq9AV6cZPdjyFd069n3dzlt0OU2pdgBje2nx6TM/VNA0NLX3wOExw5HDDk1zY1bEPT+9+ASo0\n/GPdl7CsfPGEvTYFco4tr/Uhoaj47OjIa3sfPu3HT+v3ANDwzdvrMKPajnCcNl0YranlDvzzvYvx\n8J0LMclrxcd7W/Cvz2zGGx8cRziau5Gt6UAGgOO9DdjZvntUz6PBW8Up3X88XA05jWM53Dz9Onxj\n0VdgE6z4/fG38NSuX6JXzq85y9ddPAVOm4j1W0+jq3f0H3Db/RH0heMFt6HE5pbteG7vS+BYDg8u\n/DLqSudO6OsXxo4EBWx5bRl+99FJbD3YjkvnVwz5GE3T8PM/7EcsoeLBW+dj/jQPfnvkD/ioeRNW\nVCzFFy03A6BNF86HYRgsmO7B/GlufLKvBb/78AT+vOkUPth1BjdeNhWrFleCz+IavQBQanbDxJmg\nQYWiKvjdsbewas7FIz6nuTOEn9bvRkdPFEtne7H2+rnUX1wkOlKB7DOPbt3mOe6Z+O7yb+Glg7/F\n/q5DeHzrf+BLc+/GPM+cXBZz1CSRw+2fn45f/vkg6j84jgdumgdV09AbjKG7L4qu9FdvFN19yWUy\nu/uiCKdaBAup//i9xo9Rf3QdrLwFX1/0ZUx1ZG8vgtGiQM6xMrcFU8rt2H+yGwlFHTIQGIbB/Glu\nfLqvFUebenHRbB+Wli3Eoe4j+OTMVmxu3YHLKpbj2qmr4ZRogNf5sCyDlQsmYXltGTZub8SfN53C\nbzYexcbtjbj989OxbI4vazvasAyLavskHOs5icurLsUHTZ9g3aENuKLs80M+fs/xLvxi3T5EZAU3\nXTYVN31uGvUXF5H2zJSn0W+kYBdt+NqCv8f7TZ/g98f+jP/b3n3HR1Xmix//nGmZkknvjTRqQgu9\nd1xFsS2IilxWXPVnQV1dRO9e4f7299q7ru69P11317L2sqviriIWkCaIEDFIIKGGJBCSkN4nybRz\n/5hkkpAEkjDJzITnzSuvZM6ZOfM8PDPne57nPOUvmW8wP3YWS5OuRa0Y+FO0xWqjorbZGWjLW2rG\n6cdKSD9WglIhYeum57WPRkmIn5akaH/CAnXMGhM1kEnvE1mW+Tp/B1vytuGvMfLQuF8S5RvhlrSI\ngDwA7lg4lBNnq1Aquj/xrlw8jLziWrYdLGBojD8Thsfz75N/xY8lh9l6bgd7CvfzffFBZkVPZfGQ\nefhpvOfK01181EqWTItn1tgotuzLZ9dPhbz8WTZbfyhg+bwkly2PGWuM5nR1LinBI/ip9AifndjG\nGP8xBGnbji9m3ro6lJrKkZAIvsQ95K5IksS82JkkByTyZvb77CzYy+nqXH6Rcgfh+lCXpU+WZRqa\nrFTUtNVsW2u5lS2Pa02XnnQnLtxIsL+WED8tQX4+BPtrCfbTEuyvRe+jcvnyjf1JlmX+mbOFnQV7\nCdYG8vC4ewnV967sXEmS+3Nl6stw9Yod3r4KSGF5A799+yBKhcSG1ZOc09YFBuvZcnQXX+XtoKq5\nGo1CzZyYGSyMm4OvxuDmVPfdQJdXSZWJf36by8ETjslaxiWHcOvcJKJDruz/8IcLh3j72D9YNvRG\ntCof3j3+ERPCxnJ36p2Ao/PWe9tOsiezGH9fDQ/fMsYr7xd7+/erO67M11Pf/Ra1QsX/nf5Un4/R\nbDOz6dRnfF98EI1Sw/JhNzE1YkKPAp3NbqeqrpmKmiYsSOSfr+7UrNzdvAgqpUSQX0twbQmwQX4+\njsDrr+XDHTkczinn3qWjmDrKPTVIcF152WU7fz/xCd8XHyRCH8bD4385IC2Ql1rtSdSQPUh0iIFV\n1wznb1uO85dPs/j3uyagVilRKZTMiJrClIgJfF90kK1nd/LNud3sKfyeeTEzWRA3G7366pzFqzfC\nA/X8n5tSWVxUw8e7znA4p5zMM+XMGhPFjTMT+rxgR1xLx66CukLuHPlz9pf8QEZpJrOrpxOuiebP\n/8riVLtlE8VkH4NTk7WJWnMdIwKHXtFxfJQa7hy5jBFBQ/ngxD957/hHnKg8xYrht4BN1e7ebXNL\nkG2ivKWGW1XXTHdVLINWRXigriXQtgVdRwD2wWjQXPL2yYqFQ8nKq2DT7jOkDQ316jXLrXYrbx37\nBz+VHiHOGM2DY+/xiMqNCMgeZnpqJKcKqtmTWczft59m1c/aOneoFCpmx0xjWuREvitKZ+vZnXx9\ndiffFn7P/NhZzIudhU4lTvaXkxTlz5N3jCczp4KPd+ewJ7OIA8cusHhSHNdOiUPn07uvRZg+FI1C\nTUF9IQpJwerxy/jNjuf44Ni/qD8yhfLqZtF56yrgHPLUi/vH7dllmboGc0twbaaiJohRzTdyjB38\nWHKYH8+doilnDHJD5+GQkgRBRh+So/2dQXZIdAAaCUcANvr0+nN9sbAAHYsmxvJV+jm2Hizghunx\nV3Q8dzHbzLx29F2OVZ4kOSCB+8f8wmPOmyIge6A7Fg4jr7iO3YeLGBobwNK5HZs41Eo182JnMiNq\nMnsK9/PN2d18kfcNuwv2sSBuNnNiZqBVicncL0WSJMYNDWF0UhDfHSnm072Otaa/PVzI0hkJzBkX\n1eMe2QpJQYwxivzaAirqGqio0xJqT6akKQezOoelM2aJzltXgcsNebJYbVS2NCe3b0Jufw/Xauui\neiuloY3NRQrPQTsqnVhbGimGyYQE6JzNywFGDUpFx89rf9xiuH56PN8dLebL/WeZOTrS65aBbbQ2\n8tfMNzlTk09K8AjuSb2rw4xq7ibuIXuokkoT//nWQWQZ/uexOWgvERuarE3sPv89O859i8naiK/a\nwKIhc5kdPQ2N0nMH5XtSeTWbbWw9eI6v0s/RbLYRHqjj1jlJTBgeesl7d9X1zZw8V83Wwq8oUR6j\nKXsacoM/qJvQjtmLTq3h/81cj07l/at0eVJ5uVJrvmx2O2aLHYvNjtVqx2y1Y3H+2Nr+trU9r/2+\nHMuP5PEjQy2LMJijHfstNmpNFiprm6hpMHebBj+Dxtl03Nqk7Og05WhWNmhVnK4+w1vZ/6DGXMvw\nwGRWjbrtkvc8+6u8dh8u5J2vTzJzdCR3Lxnp8uNfTl/zVWeu58+H/0ZBfRETwsayatRtqNzQi/1S\n95BFQPZgB0+U8tdPs4gMNjA6MQijXo1Rr+n4W6dB56NEkiQarY3sLPiOnef20mRrwk9jZPGQecyM\nmoLag64CW3liedU2mNm8L49vDxdhs8skRfuxbG4yw2IdzYSVtU2cLKjm5LlqThZUU1JpAkAZch5N\nYhYhdZOYGz+TmGAdp80ZfJG/lQWxs7ll6PXuzJZLDER52eztg6D9oiBoaxcEO/6YWwNmN/st7fab\nL9pnbQmwV7pAiTrhCKrQIpqOzEJuarsfqVJKBBnbOkkFX3T/NsjPB7WqZ7cy6s0NvHfiY46WH8NX\nbWDlyGXdTl7RX+Vlt8tsfPMHCssaeGb1pAGfGrMv+apqquZPh/9GiamUGVFTWDH85gGbDvNiIiB7\nsY925fB1+qWnrVMpJXx1jiDtp1ej08tU649TLGVjl6z4YCBJNYF49SjUKjUqpYRSIaFSKlAqJVSK\nlt9KBSqFhLLddpXS8bh1u+O1jt8qpQLFJYZyXY4nl9eFShOffHuGjJOOGdaGxwZQVddMaXXb7Gla\njZKhMQEMjwsgKLSZd8/+jRlRk3lk1i8oK6vDYrPw2/TnqWqu4alJj7ptbGNv2e1yh+DWGux8jVpK\ny+p7HCQ7BMEuapQX77Na7d2Ob3UFCVCrFBf9KNFrHbUktdKxTXPRfrVK0WmfSqVA07qvZf8nRe9S\n3FjI+jH/jk6tce7z0ShdertClmX2FO7nnzlbsNqtzI2ZwU3JSzqNWe7P71d2fiV//MdhhscGsO6O\n8QM61Km3+So1lfOnw69R2VTFwrg53JR0nVuHZomA7O1UKvIKKqkzmakzWVp+zG2/G9seN5nbDWlQ\nmVFF5qEKO4uktGNv1mItTMZWEQWya64OJYkOAdoRyFuDd2sgbwvqKqXCeTFg0GuwWGwXvbbdxUFL\n8O94sdB6jNaLhdYLB8fzWo/tPOZFFxJKpdSrk2PO+Ro+2pVDTmENOh8Vw2L8GR4XyPC4AOLCfZ33\n7Wx2G7/a8x9EGcJ5/rrfOD+HmWVZvHr0HQxqPfePWU2if3yP3tcuyz0Kbtb2tcNONcrW53Xeb273\nPKvV1qHW2J9BEWgLcOr2gU7ZRbB07G/dp1JdFDCdx2h7raaLgKtWKtCoHZ+Frk7ErjpvPLn3P9Gq\ntPzntCev+Fg9UVhfzBtZ73PBVEq0byR3p9xJhCHMub+/z4cvfJxJ5pkKHrx5NBOGu26s9OX0Jl+F\n9cX86fBr1JnrWZr4MxYPmef2cdJi2JOXCw3UgbVn41YtVht1JgumJitWux2rbRq1zbUcrNxPdu1P\nKBKzMA49z2jDVGLUw5Dtjll3rDZH053NJmO1y9hsdqw2Gavdsc1ms2NteZ6t3Xarze58ffvHZosN\nU5MVW7tjuWvN4ospWy8SFF0HbmfLQMs2rY+S+AgjwX5a1CoFheX1lFSZOrUy6OUgztcV8/HOE9TW\nmlsCnA+Jtpnkyvv47x9fJrJhOrqmuC7uUbYEztZA21XnHhdSXVTj89WpOwRBtUp5UW1QgZ9Ri9Vi\nde7vOgh2XaNs3adSdh0UvV2jtZF6SwNxxpgBe89o30ienLSWTac/Z19ROs8efIFlw25kWuSkAfk/\nXj4/may8Sj7elcOYpGDUKs9aGiGv5ix/yXwDk7WR5cNuYk7MdHcn6bJEQB5k1ColQX5KgjrEb3/S\nEmOpalrMtrO72Ff0A9/Xfk24/hBLEhYyPmzMgNxPsctyh6DtH6CntLSu5cLBEbjbLg7aLgKcFwct\n+y++WOh0UeD8u3V714+tNhmbve0Cw9xs7fSe7eVfuPRVuTreB1WYnXd3HUQ2tS8AXxR+E9AkH6bI\n9zssVcOwFie0BMW2AKbXqrsObl3WKC/a3ykIKjsE0/bBU6lU9KkJddC2QLlATxeVcDWNUsMdI25t\nGbP8Ce+f2MTxylPcPvxWoH/v7UYGG5g3PprtGefZkXGen00Z+Lmfu3Oi8jSvHH0bq93KqpG3MSVy\ngruT1CMiIF9FArUB3Db8ZhbGzeXr/B0cuPAjb2R/QFT+TpYkLGJsaGq/XlkrJAmFSnJeSQf5abE1\nX3qaPneSZUet3nlxYJM7tSa0f3ykGnaXF3DjNcEk+6RcFEyVVDRP4d3T71ETe4rZk/25ffgtKBVi\nXPJg0NtFJVwtLWwMQ4yxvHXs7xwqPUJ+bQGPzVhDEGGXf/EVWDozgf3ZF/j8+zymj47wiKUWM8uy\neSPrPQDuSV3J2NBUN6eo55QbN27c6K43N5m6HwbQFwaDj8uP6QlcnS+9WseY0FFMCh9Po7WJk1U5\nZJRmcrT8GP4+foTpQgakycvTy0uSHPebW2uyPholOh8VBm1LBzqDhgBfH8cQFX8dRp2afUXpJIVH\nMGvIWMf4UF8f/AwaDFo1ob4BTAgfy+nqXLIrTpBfW8DokJGoFZ7XA74rnl5efeWKfB0uy+J0dS7z\nYmcOeC25lV6tY0pEGiCRVX6c3fkHUEgSif7x/fZ91qgdLTE/nS6n2WxjbHL/5/1S5ZVenMFbx/6O\nUqHi/jG/IDVk4IdlXY7B0P3Ybc9q9BcGVKg+mFWjbuM/pjzOxPBxnK8v5uUjb/FcxkscqziJG/v7\neaVI3wgUkoK8qoJunxPg48+j4+8nNXgkxytP8d8Zf6WyqWoAUyn0h9Ym677O0uUqSoWS6xMX88j4\newnU+vN57lZe/OlVqptr+u09542PJiJIz7eHi7Da7P32Ppdis9vYXbCPd45/iI/Sh7XjfsmIoCub\nwtQdRA3ZC/R3vnw1BsaHjWZc6GjqzPWcrDrNwZKfOFGVQ4gukGBdUL+872ArL6WkILMsi6L6CyyK\nm9vtfXmVQsWE8LGYrCayKo5zqCSToYFJ+Pt49oITg628WrkiX9vO7abe3MBNSde5bXxre8G6IK5L\nmUN+RRHHKk+SXpxBmD6UcP2lJ7rpC4VCYlR8IENj/IkN6/8xybLKyrELp8ksz2Zv4QG+zNvOJ6c/\nJ6viOEaNL4+Mv484v4HrXNdbl6ohi3vIglOUbwS/HH0XBXVFfJG3jaPlx3jhp1cZFpDEksTFJAck\nuDuJHi/WGM35+iJKTGWXHHeskBQsH3YToboQPjn9Of9z6GXuTrmj20keBM9WZionRBfkUX0CfH0M\n/DL1Lr4rOsAnpz/n1aNvE+DjT3JAAsMCkkgOTHTZ7anIYAORwa5dnMEu2ylvrOB8fTGF9cWcryui\nsL6YqubqDs9TK9REG6OI9Y1i0ZC5hPRy6UtPIgKy0EmsMYr7x6zmbG0BW3K3cazyJKcO/ZWRQcO4\nPnEx8X6e05vS08Qao9lffJCCusIeTQQyL3YmQdoA3sz+O68ceZufD1vK3JgZA5BSwVXqLQ2YrI09\nHmM+kCRJYlb0NJL8E/gqfzunqs44FqooOQyAn8bI0IBEkgMSGRaYSLg+zC3D0pptZorqL3C+vsgZ\nfIsaimm2dWy58NcYGR+ZQqgmjGjfSGJ8owjTh3hEq4QriIAsdGuIXywPjltDbk0+W3K3cbzyFMcr\nT5EaPJLrExcT27LsoNAmtt1SjD0dajE2NJXH0u7nr5lv8vGpzyhvrOCW5OsHzUlmsCvzkPvHlxLl\nG8Ga1JXIskyJqZRTVbnkVOdyujqXjNJMMkozATCqfUkOSCA5MJGhAYlEGsJd+jmUZZkacy3n64pa\nar5FnK8vosxUgUxbnxWFpCBCH0a0bxQxRkfgjfaNxKjxHdTD70RAFi4r0T+etePv5XTVGT7P3UZW\nxXGyKo4zNjSVJQmLiPaNdHcSPUaMbySSJHGurrBXrxviF8uvJz7EX468ya6C76hsrGJ1yu0evTiI\n4OAcg+ymIU+9IUkSEYZwIgzhzI6ZhizLlDaWk1OVy6nqM+RU5/FT2VF+KjsKgEGtJzkg0VmLjm7p\nuNgTVruVElNZS/BtqfnWF9FgMXV4nk6lIzkgwRl0Y4xRRBjCO00FejW4+nIs9NnQwCQeS7ufk1U5\nbMndSmZZFkfKskkLG8N1CYs6TNt3tdIoNcQYIzhfX4hdtveqdhGsC+LxtAd4LetdMsuz+f+HXuG+\nMavx9xnYyfuF3ilr9PwacnckSSK8pbPXjOgpyLJMeWMlp6tzOV19htNVuWSWZZFZlgW0Bc+hLUE6\nxhiFQlLQYDE5arvOmm8xxQ0l2GRbh/cL0QU7XtcSfKN9owjSBgzK2dv6QgRkoVckSWJE0FCGByaT\nXXGCL/K2kVGayaHSI0yKGM+18Qu98sTkSgmBcRTUFlPWWEG4vndz/OrVOh4cezd/P/FPDlz4kecz\nXuKBsXcTaQjvp9QKV8qbasiXI0kSofpgQvXBTI+aBEBFa4CucjRxHy0/xtHyYwBolVq0Kp9Ow6rU\nCjUxxihifFubm6OI8o1Ap9IOeJ68iQjIQp9IkkRqyEhSgkdwpDybL/K+4YcLh/ix5DBTIiZwbfyC\nfhsu5ekSAmPZczadgrrCXgdkcAyLWjlyGSG6YLbkbeWPGX/ml6mrGB6U3A+pFa5UWWM5KoWKQG33\naxN7s2BdEMG6IKZGTgQcSxm2BegzWOxWRgUPJ8bXEYCjB1lHq4EkArJwRSRJYmxoKqNDRnG4LIsv\ncrexv/gg6RcymB45iZ/FLyBQG+DuZA6ohEBHL/SCukImho/r0zEkSeLahAUE6wJ5//jHvJT5N+4c\n8XPnSVHwDLIsU2qqIEQXfNUEoEBtAJMj0pgckebupAw6IiALLqGQFKSFjWFcaCo/lhzmq7ztfFeU\nzoHiH5kRPZVrhszz+IkvXCUhMBag1x27ujI5Io1AnwBePfo27x7/iPLGCpYkLBb33DxEvaWBJlsT\nYbokdydFGASujks6YcAoJAWTI9L4zZTHWTlyOf4+/nx7fh8b9v+ef57eQp253t1J7Hc6tZYwfQgF\ndYUumX50aGAiT0x4kBBtEF/l7+DtY//AYre6IKXClWpb5cl7J6MQPIcIyEK/UCqUTIucyIapv+b2\n4bfgq/ZlR8Eentn/ez478xX1lgZ3J7FfxfpG02htpMJF81SHG8J4YuJDJPjFcbDkJ146/Fqn4SPC\nwCttdO8qT8LgIgKy0K+UCiUzo6eyYdo6lg+7CZ3Sh21nd7Hh+9/zUdbnmCyN7k6iy2WVnOB0dS4A\ntWbXTWBg1Piydvx9jA8bQ051Hs9nvESZqcJlxxd6zxsmBRG8hwjIwoBQK1TMiZnOxmnruTX5etQK\nNZuyv+SZ/b/nq7wdNFmb3J3EK2az29iSu5Xf7n6ReksDNycvIcHF04xqlGruTrmDRXFzKTWV83zG\nS+TWnHXpewg911pDHgxDngT3E526hAGlUaqZHzebGdFTyajK4NPjW9mSt5Vd5/eyKG4us2Om4+OF\ns1NVN9fwZvYH5FTnEWYI5t9G3t5vc34rJAU3JV9HiC6ID099ygs/vcK/jVpBWtiYfnk/oXtlpnLU\nCvVV02FR6F8iIAtu4aPUcOPIxYwPGM/ugn3sKNjDp2e+ZMe5PSyOn8fMqKlolGp3J7NHssqP887x\nD2mwmBgfOpq1M1djqrFd/oVXaGb0VAK1gbye9S6vZ71HedK1zI+dheoqnHLQHVqnnQy9ioY8Cf1L\nfHMFt9KptFybsIA5MdPZWbCXXQV7+eT053xzdjeL4uYwM3qqx87nbLVb2Xzma3YU7EGlUHHbsJuZ\nFT0Vg0aPiYGZ/D4leDi/SnuAvx55k8/OfMWXed8wxC+WRP94kvzjSfQfgl6tH5C0XG1qzXWYbWZx\n/1hwGRGQBY+gV+u4PnExc2NnsOPcHr49v49Pcraw7exuFg6Zw6zoaR7VlF3eWMkb2e9ztraAMH0I\na1JWEmOMcktaYoxR/HriQ2w7u4uc6jzOVOeTU53n3B9hCCfJfwiJ/vEk+scTqgsW45hdYDBNmSl4\nBhGQBY/iqzZwY9K1LIibza6C79hdsI9/5XzBN2d3syBuNrOjp6F183y4h0qP8MGJTTRam5gckcZt\nw25Gq/Jxa5oCfPxZPuwmABqtjeTVnCO3Jp8zNWfJrz3HhYYS9hX9ADh6aye21J6T/OOJNUaLZu4+\n8OZFJQTPJL6FgkfyVRu4IfEaFsTOYlfBd+w6/x2fnfmK7We/ZX7cbObETB/wieotNguf5Gxhb+F+\nNAo1d41c7pFTWepUOkYFD2dU8HDA0fu7sL6YMzX55Nbkk1tztsMKPmqFijhjLEkBjiCd6B+PQTRz\nd9BobeRCQynFDaVcaCih2FTCudrzgKghC64jArLg0fRqPUsSFzM/bha7C/axs2Avn+d+zfZz3zI/\ndiZzY2aiV+v6PR0lDaW8nv0+hfXFRBkiWJN6JxFesgKTUqEkzi+GOL8Y5sXORJZlKpuqW4JzvjNQ\nn6lp18ytD3PUogPiSfIfctUEHZPF1CHoFteXcMFU2mk1IwB/jR/jQlOJ94t1Q0qFwUiSXTG3Xx+V\nlbm240toqNHlx/QEIl9tGq1N7Dn/PTsK9tBgMaFTaZkbM5N5sTP7rVaXXpzBP079C7PNzMyoKdw6\ndOkle4B7Y3k1WpvIrznnDM55tecw28zO/Ua1LyPCkojRxZDY0sztzQvI15sbKG4o4YKphGpbFbkV\n57nQUNLlRC6BPgFEGMKINIS3/daHeXxnOW/8HPaEt+crNLT79c1dHpB/97vfkZmZiSRJPP3004wZ\n0/3YSBGQe0bkRcc9iAAACyhJREFUq7MmaxN7Cw+w/dy31Fsa0Cp9mBszg3lxs/BVG1ySvmabmY9O\nfsqBCz+iVfpwx4hbmdCD1ZsGQ3nZ7DYKG4rJrT7rrEW3ryWqFCqGGB3BOSkgngT/IS77f3cVWZap\ns9RTXO+o7V5orfk2lHQ5dWuwNpCI1qCrD3f+7a1r+A6Gz2FXvD1fAxaQf/jhB15//XVeeeUVzpw5\nw9NPP82HH37Y7fNFQO4Zka/uNdvM7C3cz/az31JnqcdHqWFoQBI6lQ69WotOpUOn0qJX6dC2/Nap\nHNtb/1YqlJ2OW1hfzOtZ71NiKiXOGM3dKSt7vIDAYC0vyWDhYG62o6m7Op/z9cXItJ0+wvVhBOsC\n0Sg0aJRqNAo1aqXa+VitULds17RsV7ds73q/SlL2qDe4LMvUmGsdNd6G0pbfjr8brB3n+5aQCNYF\nEWkII0IfTqQhnJExCWiaDW7vmOdqg/Vz6O35ulRAdmmb0/79+1m4cCEASUlJ1NTUUF9fj6+vryvf\nRhCcfJQaFsbNYXb0NL4rSmf72d1kVRzv1TE0CrUjcKt16JRadCqtc+H1ebEzuTHpOq9unnWVEH0Q\nE8PHOdd4brI2kV9b4Gjmrs4nr/YsJaZSl72fhNRNAFejUWpQK9TUmeu5YCqh8aKpVyUkQvXBJAck\ntNV6DRGE60M73W4IDfLuE7wweLj0LFNeXk5KSorzcVBQEGVlZd0G5MBAPSpV59rJlbjU1Yc3E/m6\nvNsirmP5+GtptDZhMjfSYDHRYG7EZHH8NJhNNFgaMbX+tjRianlOg6WRBnM9paYy7LIdo8bAY5Pv\nYWL0WLfny5N0zJeR2MhQZuFYqN4u2zHbLJitZpptZsw2C81WM2Zb58etfzu2mzFbLTS3Ps/asv+i\nvxtsJqqaHY9ba+ZKSUGEMYwYv0jHj38EsX5RRBjDejXT29VRXoPHYM1Xv172X641vKrKtcvHeXtT\nRndEvvpCjR5/9JI/aHD89OAWpyzLmO0WVJISpULZp/SJ8gLQtPxr+U9Xtvy4YG4XWZax2q2Y7RZ8\nlJrOY6gtUFPZBPRswRJRXt7F2/M1YE3WYWFhlJeXOx+XlpYSGhrqyrcQhH4lSZJHzQgmdCZJEmql\n4/60IAwmLp0RfcaMGWzduhWA7OxswsLCxP1jQRAEQegBl9aQ09LSSElJYcWKFUiSxIYNG1x5eEEQ\nBEEYtFx+D/mJJ55w9SEFQRAEYdATi3gKgiAIggcQAVkQBEEQPIAIyIIgCILgAURAFgRBEAQPIAKy\nIAiCIHgAEZAFQRAEwQOIgCwIgiAIHkAEZEEQBEHwAC5dD1kQBEEQhL4RNWRBEARB8AAiIAuCIAiC\nBxABWRAEQRA8gAjIgiAIguABREAWBEEQBA8gArIgCIIgeACXr4fsLr/73e/IzMxEkiSefvppxowZ\n4+4k9dkf/vAHMjIysFqt3HfffezcuZPs7GwCAgIAWLNmDXPnznVvInspPT2dRx55hKFDhwIwbNgw\n7rnnHtatW4fNZiM0NJTnnnsOjUbj5pT2zscff8zmzZudj7OyskhNTcVkMqHX6wF48sknSU1NdVcS\ne+3UqVM88MADrF69mpUrV1JcXNxlOW3evJm3334bhULB8uXLWbZsmbuT3q2u8vTUU09htVpRqVQ8\n99xzhIaGkpKSQlpamvN1b731Fkql0o0pv7SL87V+/fouzxXeVFbQOV9r166lqqoKgOrqasaNG8d9\n993HDTfc4PxuBQYG8uKLL7oz2VdOHgTS09Ple++9V5ZlWc7JyZGXL1/u5hT13f79++V77rlHlmVZ\nrqyslOfMmSM/+eST8s6dO92csitz4MAB+eGHH+6wbf369fKXX34py7Is//GPf5Tff/99dyTNZdLT\n0+WNGzfKK1eulE+ePOnu5PRJQ0ODvHLlSvk3v/mN/O6778qy3HU5NTQ0yIsXL5Zra2vlxsZGecmS\nJXJVVZU7k96trvK0bt06+YsvvpBlWZbfe+89+dlnn5VlWZYnT57stnT2Vlf56upc4U1lJctd56u9\n9evXy5mZmXJBQYF88803uyGF/WdQNFnv37+fhQsXApCUlERNTQ319fVuTlXfTJo0iRdeeAEAPz8/\nGhsbsdlsbk5V/0hPT2fBggUAzJs3j/3797s5RVfmz3/+Mw888IC7k3FFNBoNr732GmFhYc5tXZVT\nZmYmo0ePxmg0otVqSUtL49ChQ+5K9iV1lacNGzZwzTXXAI6aVXV1tbuS12dd5asr3lRWcOl85ebm\nUldX59UtoJcyKAJyeXk5gYGBzsdBQUGUlZW5MUV9p1QqnU2dmzZtYvbs2SiVSt577z1WrVrFY489\nRmVlpZtT2Tc5OTncf//93H777ezbt4/GxkZnE3VwcLDXlhnAkSNHiIyMJDQ0FIAXX3yRO++8k2ee\neYampiY3p67nVCoVWq22w7auyqm8vJygoCDnczz5O9dVnvR6PUqlEpvNxgcffMANN9wAgNls5vHH\nH2fFihW8+eab7khuj3WVL6DTucKbygq6zxfAO++8w8qVK52Py8vLWbt2LStWrOhw68hbDZp7yO3J\ng2A20O3bt7Np0ybeeOMNsrKyCAgIYOTIkbz66qu89NJLPPPMM+5OYq/Ex8fz0EMPce2111JQUMCq\nVas61Py9vcw2bdrEzTffDMCqVasYPnw4cXFxbNiwgffff581a9a4OYWu0V05eWP52Ww21q1bx9Sp\nU5k2bRoA69atY+nSpUiSxMqVK5k4cSKjR492c0p77sYbb+x0rhg/fnyH53hjWYHjYikjI4ONGzcC\nEBAQwCOPPMLSpUupq6tj2bJlTJ069bItBp5sUNSQw8LCKC8vdz4uLS111lS80d69e3n55Zd57bXX\nMBqNTJs2jZEjRwIwf/58Tp065eYU9l54eDjXXXcdkiQRFxdHSEgINTU1ztpjSUmJV3+R0tPTnSe+\nRYsWERcXB3hvebWn1+s7lVNX3zlvK7+nnnqKIUOG8NBDDzm33X777RgMBvR6PVOnTvW6suvqXDEY\nygrg4MGDHZqqfX19ufXWW1Gr1QQFBZGamkpubq4bU3jlBkVAnjFjBlu3bgUgOzubsLAwfH193Zyq\nvqmrq+MPf/gDr7zyirOn5MMPP0xBQQHgOPG39lT2Jps3b+b1118HoKysjIqKCm655RZnuW3bto1Z\ns2a5M4l9VlJSgsFgQKPRIMsyq1evpra2FvDe8mpv+vTpncpp7NixHD16lNraWhoaGjh06BATJ050\nc0p7bvPmzajVatauXevclpuby+OPP44sy1itVg4dOuR1ZdfVucLby6rV0aNHGTFihPPxgQMH+K//\n+i8ATCYTJ06cICEhwV3Jc4lB0WSdlpZGSkoKK1asQJIkNmzY4O4k9dmXX35JVVUVjz76qHPbLbfc\nwqOPPopOp0Ov1zs/hN5k/vz5PPHEE+zYsQOLxcLGjRsZOXIkTz75JB9++CFRUVHcdNNN7k5mn5SV\nlTnv0UmSxPLly1m9ejU6nY7w8HAefvhhN6ew57Kysnj22WcpLCxEpVKxdetWnn/+edavX9+hnNRq\nNY8//jhr1qxBkiQefPBBjEaju5Pfpa7yVFFRgY+PD3fddRfg6Ay6ceNGIiIi+PnPf45CoWD+/Pke\n3Xmoq3ytXLmy07lCq9V6TVlB1/n605/+RFlZmbPlCWDixIl8+umn3HbbbdhsNu69917Cw8PdmPIr\nJ5ZfFARBEAQPMCiarAVBEATB24mALAiCIAgeQARkQRAEQfAAIiALgiAIggcQAVkQBEEQPIAIyIIg\nCILgAURAFgRBEAQPIAKyIAiCIHiA/wX9g6OpabyU4AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7faddc220ef0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "FJEk8gg3ASTG",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Sub-sampling(for faster run) and reformat**"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "AsYZrNVwulEt",
        "outputId": "56c238fa-9c0d-4a30-c1e0-02b140dd097e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "cell_type": "code",
      "source": [
        "#sub sample for experiment due to memory limit\n",
        "train_size=20000\n",
        "test_size=4000\n",
        "\n",
        "train_sample = train_data_raw[0:train_size]\n",
        "train_label_sample = train_label[0:train_size]\n",
        "\n",
        "test_sample = test_data_raw[0:test_size]\n",
        "test_label_sample = test_label[0:test_size]\n",
        "#train_sample = random.sample(list(train_data_raw),train_size)\n",
        "#test_sample = random.sample(list(test_data_raw),test_size)\n",
        "print(len(train_sample))\n",
        "no_point = 0;\n",
        "\n",
        "\n",
        "def get_max_len(sub_sample):\n",
        "    max_stroke_length = 0\n",
        "    max_point_length = 0\n",
        "    for i in range(len(sub_sample)): #each image\n",
        "        if len(sub_sample[i])> max_stroke_length: \n",
        "            max_stroke_length = len(sub_sample[i])\n",
        "        for j in range(len(sub_sample[i])): #each strokes\n",
        "            if len(sub_sample[i][j][0])> max_point_length:\n",
        "                max_point_length = len(sub_sample[i][j][0])\n",
        "    return max_stroke_length,max_point_length\n",
        "                \n",
        "max_stroke_len,max_point_len = get_max_len(train_sample)    \n",
        "print (max_stroke_len,max_point_len)\n",
        "    \n",
        "# no 0\n",
        "def process_data(sub_sample, append):\n",
        "    stroke_data = []\n",
        "    point_data = []\n",
        "    for i in range(len(sub_sample)): #image\n",
        "        for j in range(len(sub_sample[i])): #strokes\n",
        "            for k in range(len(sub_sample[i][j][0])): #points\n",
        "                if append:\n",
        "                    if k == len(sub_sample[i][j][0])-1:\n",
        "                        temp2 = [sub_sample[i][j][0][k],sub_sample[i][j][1][k],1]\n",
        "                    else:\n",
        "                        temp2 = [sub_sample[i][j][0][k],sub_sample[i][j][1][k],0]\n",
        "                    point_data.append(temp2)\n",
        "                else:\n",
        "                    temp = [sub_sample[i][j][0][k],sub_sample[i][j][1][k]]\n",
        "                    point_data.append(temp)\n",
        "        stroke_data.append(point_data)\n",
        "        point_data = []\n",
        "            \n",
        "    return stroke_data\n",
        "\n",
        "# augment with 0\n",
        "def unpack(x,max_strock,max_len):\n",
        "    x_new=torch.zeros(torch.Size([x.shape[0],max_strock,3,max_len]))\n",
        "    for i,item in enumerate(x):\n",
        "        for j,strock in enumerate(item):\n",
        "            strock=torch.Tensor(strock)\n",
        "            x_new[i,j,0,:len(strock[0])]=strock[0]\n",
        "            x_new[i,j,1,:len(strock[0])]=strock[1]\n",
        "            x_new[i,j,2,:len(strock[0])]=0\n",
        "            x_new[i,j,2,len(strock[0])-1]=1\n",
        "    return x_new\n",
        "\n",
        "# no 0\n",
        "train_data = process_data(train_sample, 1)\n",
        "test_data = process_data(test_sample, 1)\n",
        "\n",
        "print (len(train_data))\n",
        "print (len(test_data))\n",
        "\n",
        "# with 0\n",
        "train_data_amend = unpack(train_sample,max_stroke_len,max_point_len)\n",
        "test_data_amend = unpack(test_sample,max_stroke_len,max_point_len)\n",
        "\n",
        "print (train_data_amend.shape)\n",
        "print (test_data_amend.shape)\n",
        "\n",
        "print(train_data_raw[0])\n",
        "print(train_data[0])\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "20000\n",
            "61 556\n",
            "20000\n",
            "4000\n",
            "torch.Size([20000, 61, 3, 556])\n",
            "torch.Size([4000, 61, 3, 556])\n",
            "[[[30, 24, 23, 25, 33, 68, 131, 151, 160, 163, 164, 150, 128, 79, 49, 37], [90, 81, 41, 34, 26, 12, 0, 17, 32, 48, 99, 114, 118, 117, 103, 85]], [[58, 43, 21, 10, 1, 0, 14, 27, 158, 166, 193, 193, 189, 183, 173, 133, 62, 56, 50], [243, 241, 234, 225, 210, 168, 145, 140, 132, 135, 180, 220, 233, 240, 247, 255, 251, 250, 241]]]\n",
            "[[30, 90, 0], [24, 81, 0], [23, 41, 0], [25, 34, 0], [33, 26, 0], [68, 12, 0], [131, 0, 0], [151, 17, 0], [160, 32, 0], [163, 48, 0], [164, 99, 0], [150, 114, 0], [128, 118, 0], [79, 117, 0], [49, 103, 0], [37, 85, 1], [58, 243, 0], [43, 241, 0], [21, 234, 0], [10, 225, 0], [1, 210, 0], [0, 168, 0], [14, 145, 0], [27, 140, 0], [158, 132, 0], [166, 135, 0], [193, 180, 0], [193, 220, 0], [189, 233, 0], [183, 240, 0], [173, 247, 0], [133, 255, 0], [62, 251, 0], [56, 250, 0], [50, 241, 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "6t9V5ZoNWOZd",
        "outputId": "42b219f3-1540-485b-bf15-28362967f581",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "#string to int label\n",
        "label_set = set(train_label_sample)\n",
        "label_list = list(label_set)\n",
        "train_label_tensor = []\n",
        "for i in range(len(train_label_sample)):\n",
        "    train_label_tensor.append(label_list.index(train_label_sample[i]))\n",
        "test_label_set = set(test_label)\n",
        "test_label_list = list(test_label_set)\n",
        "test_label_tensor = []\n",
        "for i in range(len(test_label_sample)):\n",
        "    test_label_tensor.append(test_label_list.index(test_label_sample[i]))  \n",
        "print (len(train_label_tensor),train_label_tensor[0],len(test_label_tensor))\n",
        "print(set(train_label_tensor),set(test_label_tensor))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "20000 3 4000\n",
            "{0, 1, 2, 3, 4} {0, 1, 2, 3, 4}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RD3rLfs9I33W",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Other data format (not used)"
      ]
    },
    {
      "metadata": {
        "id": "nwdeNv6U_L-H",
        "colab_type": "code",
        "outputId": "d35e74ef-bb7b-4b98-a175-d7e71f29a4b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "#append labels to match no.points\n",
        "total_train_label = []\n",
        "total_test_label = []\n",
        "\n",
        "for i in range(len(train_data)):\n",
        "    for j in range(len(train_data[i])):\n",
        "        total_train_label.append(train_label_tensor[i])\n",
        "        \n",
        "for i in range(len(test_data)):\n",
        "    for j in range(len(test_data[i])):\n",
        "        total_test_label.append(test_label_tensor[i])\n",
        "        \n",
        "print(len(total_train_label),total_train_label[0])\n",
        "print(len(total_test_label))"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "637360 3\n",
            "130487\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "3CvVRG6i_L-L",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#a list of all points\n",
        "train_data_points =[]\n",
        "test_data_points = []\n",
        "\n",
        "for i in range(len(train_data)):\n",
        "    train_data_points.extend(train_data[i])\n",
        "    \n",
        "for i in range(len(test_data)):\n",
        "    test_data_points.extend(test_data[i])\n",
        "\n",
        "print (len(train_data_points),train_data_points[0])\n",
        "print (len(test_data_points))\n",
        "\n",
        "# define manually according to factorization of the total no.points\n",
        "point_bs = 19\n",
        "point_len = 16777\n",
        "\n",
        "\n",
        "train_data_points = torch.LongTensor(train_data_points)\n",
        "total_train_label_tensor = torch.LongTensor(total_train_label)\n",
        "\n",
        "\n",
        "# lstm batch_first = true\n",
        "train_data_shaped = torch.zeros(point_bs,point_len,3)\n",
        "train_label_shaped = torch.zeros(point_bs,point_len)\n",
        "\n",
        "for i in range(point_bs):\n",
        "    train_data_shaped[i,:,:] = train_data_points[i*point_len:(i+1)*point_len]\n",
        "    train_label_shaped[i,:] = total_train_label_tensor[i*point_len:(i+1)*point_len]\n",
        "    \n",
        "#train_data_shaped = torch.transpose(train_data_shaped,0,1)\n",
        "print(train_data_shaped.shape,train_label_shaped.shape,train_data_shaped[:,0,:])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "t5AZJyDtA7UZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Model"
      ]
    },
    {
      "metadata": {
        "id": "qAwd4QPWbj50",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Model_2: one score for each stroke**"
      ]
    },
    {
      "metadata": {
        "id": "4e15zafUJXmY",
        "colab_type": "code",
        "outputId": "05e01e38-9afd-40e2-8013-98eac843b191",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "cell_type": "code",
      "source": [
        "#determine/adjust parameters\n",
        "m = nn.Conv1d(3, 6, 3,padding = 1)\n",
        "m2 = nn.Conv1d(1,3,3, padding = 1)\n",
        "l = nn.LSTM(3336, 40)\n",
        "p = nn.MaxPool2d(2,2)\n",
        "input = torch.randn(61, 3, 556)\n",
        "output = m(input)\n",
        "print('c1',output.shape)\n",
        "output = F.relu(output)\n",
        "output = output.view(61,1,-1)\n",
        "print('v1',output.shape)\n",
        "output,_ = l(output)\n",
        "print('l',output.shape)\n",
        "output = m2(output)\n",
        "print('c2',output.shape)\n",
        "output = p(output)\n",
        "print('mp',output.shape)\n",
        "output = output.view(-1,1220)\n",
        "print('v2',output.shape)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "c1 torch.Size([61, 6, 556])\n",
            "v1 torch.Size([61, 1, 3336])\n",
            "l torch.Size([61, 1, 40])\n",
            "c2 torch.Size([61, 3, 40])\n",
            "mp torch.Size([61, 1, 20])\n",
            "v2 torch.Size([1, 1220])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oHA03dprbkEv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class lstm_cnn (nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(lstm_cnn, self).__init__()\n",
        "        \n",
        "        self.conv1 = nn.Conv1d(3,6,3,padding = 1) #2 output channel for 1 input channel\n",
        "        self.lstm = nn.LSTM(3336, 40) # input_size = No. points in 1 stroke.\n",
        "        self.conv2 = nn.Conv1d(1,3,3, padding = 1) #3 channels for 1 h_seq\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.linear = nn.Linear(1220  , 100)\n",
        "        self.output = nn.Linear(100, 5)\n",
        "\n",
        "        \n",
        "    def forward(self, input_data, h_init, c_init ):\n",
        "        temp = self.conv1(input_data)\n",
        "        #print(temp.shape)\n",
        "        temp = F.relu(temp)\n",
        "        #print(temp.shape)\n",
        "        temp = temp.view(61,1,-1)\n",
        "        #print(temp.shape)\n",
        "        h_seq , (h_final,c_final)  =   self.lstm( temp , (h_init,c_init) )\n",
        "        temp = self.conv2(h_seq)\n",
        "        temp = F.relu(temp)\n",
        "        temp = self.pool(temp)\n",
        "        temp = temp.view(-1,1220)\n",
        "        temp = self.linear(temp)\n",
        "        temp = F.relu(temp) \n",
        "        score = self.output(temp)\n",
        "        \n",
        "        return score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3rxaHZ9fhNZw",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Build the net**"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "Jio9zcAJulE1",
        "outputId": "28bca06e-c916-46cb-d712-ba7058ea01af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "cell_type": "code",
      "source": [
        "def display_num_param(net):\n",
        "    nb_param = 0\n",
        "    for param in net.parameters():\n",
        "        nb_param += param.numel()\n",
        "    print('There are {} ({:.2f} million) parameters in this neural network'.format(\n",
        "        nb_param, nb_param/1e6)\n",
        "         )\n",
        "\n",
        "net = lstm_cnn()\n",
        "\n",
        "print(net)\n",
        "\n",
        "display_num_param(net)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "lstm_cnn(\n",
            "  (conv1): Conv1d(3, 6, kernel_size=(3,), stride=(1,), padding=(1,))\n",
            "  (lstm): LSTM(3336, 40)\n",
            "  (conv2): Conv1d(1, 3, kernel_size=(3,), stride=(1,), padding=(1,))\n",
            "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "  (linear): Linear(in_features=1220, out_features=100, bias=True)\n",
            "  (output): Linear(in_features=100, out_features=5, bias=True)\n",
            ")\n",
            "There are 663157 (0.66 million) parameters in this neural network\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "Lt68k2gSulE5",
        "outputId": "6653d033-1a76-4264-894c-fd8aa1ba848c",
        "scrolled": false,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1122
        }
      },
      "cell_type": "code",
      "source": [
        "net = net.to(device)\n",
        "\n",
        "net.linear.weight.data.uniform_(-0.1, 0.1)\n",
        "#net.linear1.weight.data.uniform_(-0.1, 0.1)\n",
        "#net.linear2.weight.data.uniform_(-0.1, 0.1)\n",
        "net.output.weight.data.uniform_(-0.1, 0.1)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0406,  0.0544, -0.0308,  0.0666,  0.0921, -0.0778, -0.0989, -0.0212,\n",
              "          0.0200,  0.0492, -0.0372,  0.0864, -0.0569,  0.0007,  0.0205,  0.0601,\n",
              "          0.0789,  0.0890,  0.0390,  0.0603, -0.0874,  0.0478,  0.0789, -0.0727,\n",
              "          0.0524, -0.0302, -0.0386,  0.0174,  0.0748,  0.0889, -0.0292,  0.0594,\n",
              "         -0.0185,  0.0675,  0.0096,  0.0220,  0.0812,  0.0713,  0.0132, -0.0182,\n",
              "          0.0837,  0.0235,  0.0161,  0.0600, -0.0060,  0.0281,  0.0144, -0.0933,\n",
              "          0.0595,  0.0112, -0.0103,  0.0397, -0.0790, -0.0032,  0.0673, -0.0106,\n",
              "          0.0427,  0.0806,  0.0092, -0.0520,  0.0977, -0.0670, -0.0366, -0.0299,\n",
              "          0.0236, -0.0315,  0.0594,  0.0473, -0.0854, -0.0349, -0.0826, -0.0476,\n",
              "         -0.0284, -0.0172,  0.0854,  0.0061, -0.0284, -0.0776, -0.0599,  0.0644,\n",
              "          0.0157,  0.0500, -0.0403,  0.0934, -0.0577, -0.0025, -0.0730,  0.0388,\n",
              "         -0.0348,  0.0723, -0.0607, -0.0849, -0.0812,  0.0964,  0.0006,  0.0147,\n",
              "         -0.0910, -0.0965,  0.0629,  0.0166],\n",
              "        [-0.0523, -0.0042, -0.0324,  0.0010, -0.0907, -0.0323, -0.0827,  0.0454,\n",
              "         -0.0887,  0.0170, -0.0021, -0.0731, -0.0885, -0.0867,  0.0157,  0.0150,\n",
              "         -0.0298,  0.0088, -0.0986,  0.0773, -0.0837, -0.0024, -0.0342, -0.0721,\n",
              "         -0.0199, -0.0645,  0.0125, -0.0964,  0.0232,  0.0317, -0.0246,  0.0852,\n",
              "          0.0005,  0.0856, -0.0749, -0.0820,  0.0976, -0.0108, -0.0517, -0.0101,\n",
              "          0.0758,  0.0379,  0.0204,  0.0641,  0.0899,  0.0135,  0.0113,  0.0995,\n",
              "          0.0240, -0.0523,  0.0244,  0.0705, -0.0284, -0.0897,  0.0608, -0.0662,\n",
              "         -0.0888, -0.0409,  0.0079, -0.0354, -0.0064, -0.0719, -0.0063, -0.0157,\n",
              "         -0.0981, -0.0074,  0.0837, -0.0134, -0.0236,  0.0283, -0.0537, -0.0160,\n",
              "          0.0661,  0.0626, -0.0116, -0.0031, -0.0288, -0.0557, -0.0651, -0.0811,\n",
              "         -0.0095,  0.0060,  0.0922, -0.0506, -0.0498, -0.0862,  0.0843, -0.0891,\n",
              "         -0.0191, -0.0097, -0.0393,  0.0098,  0.0357, -0.0775,  0.0533,  0.0611,\n",
              "          0.0620, -0.0353, -0.0249,  0.0862],\n",
              "        [ 0.0352, -0.0180, -0.0520, -0.0980, -0.0263, -0.0854, -0.0584,  0.0469,\n",
              "          0.0183, -0.0755,  0.0631,  0.0056, -0.0080, -0.0760,  0.0614,  0.0443,\n",
              "         -0.0280,  0.0505, -0.0283,  0.0086,  0.0503, -0.0359, -0.0106, -0.0038,\n",
              "         -0.0546,  0.0494, -0.0326, -0.0874, -0.0274, -0.0050, -0.0445,  0.0164,\n",
              "          0.0366,  0.0578,  0.0235,  0.0295,  0.0457,  0.0040, -0.0956,  0.0527,\n",
              "         -0.0104, -0.0357,  0.0963,  0.0317,  0.0072, -0.0736, -0.0151,  0.0429,\n",
              "         -0.0149,  0.0794,  0.0450, -0.0870,  0.0979,  0.0527,  0.0932, -0.0986,\n",
              "          0.0842, -0.0217, -0.0114,  0.0854,  0.0750, -0.0141, -0.0398,  0.0032,\n",
              "          0.0223,  0.0191,  0.0858,  0.0321, -0.0903,  0.0367, -0.0882,  0.0703,\n",
              "          0.0521,  0.0268, -0.0923,  0.0205,  0.0914, -0.0756,  0.0049, -0.0311,\n",
              "          0.0720,  0.0435, -0.0878,  0.0263, -0.0246, -0.0480,  0.0869, -0.0126,\n",
              "         -0.0007, -0.0063, -0.0402,  0.0819,  0.0107,  0.0393, -0.0267,  0.0603,\n",
              "         -0.0546,  0.0383,  0.0326,  0.0138],\n",
              "        [ 0.0696,  0.0903,  0.0877,  0.0923, -0.0325,  0.0128,  0.0580,  0.0382,\n",
              "          0.0352, -0.0925,  0.0256, -0.0891,  0.0584,  0.0928, -0.0773, -0.0297,\n",
              "         -0.0359, -0.0291,  0.0735, -0.0056,  0.0993, -0.0398,  0.0718,  0.0047,\n",
              "         -0.0252,  0.0818, -0.0756,  0.0358,  0.0501,  0.0054,  0.0138,  0.0653,\n",
              "          0.0844,  0.0421, -0.0863,  0.0072, -0.0161, -0.0806, -0.0509, -0.0855,\n",
              "         -0.0908, -0.0859, -0.0820,  0.0452,  0.0733, -0.0434,  0.0114, -0.0951,\n",
              "          0.0269, -0.0519, -0.0047,  0.0374,  0.0243,  0.0348,  0.0065,  0.0600,\n",
              "         -0.0704, -0.0862, -0.0956,  0.0591,  0.0315, -0.0958, -0.0490,  0.0054,\n",
              "          0.0779, -0.0479,  0.0543, -0.0611,  0.0397, -0.0171,  0.0399, -0.0952,\n",
              "         -0.0378, -0.0383,  0.0987, -0.0214, -0.0261,  0.0475, -0.0619,  0.0864,\n",
              "          0.0787,  0.0386,  0.0832,  0.0178,  0.0436,  0.0755,  0.0690,  0.0625,\n",
              "         -0.0247, -0.0295, -0.0004,  0.0784, -0.0782,  0.0425,  0.0253, -0.0919,\n",
              "          0.0431, -0.0777,  0.0550,  0.0693],\n",
              "        [-0.0960, -0.0531, -0.0511,  0.0783, -0.0259, -0.0633,  0.0877, -0.0734,\n",
              "          0.0670,  0.0335,  0.0154,  0.0119,  0.0300,  0.0247, -0.0961,  0.0062,\n",
              "          0.0416, -0.0478,  0.0856, -0.0088,  0.0829, -0.0705, -0.0628, -0.0868,\n",
              "         -0.0400, -0.0481, -0.0724,  0.0017,  0.0394, -0.0441,  0.0214,  0.0321,\n",
              "          0.0596,  0.0545, -0.0256, -0.0095,  0.0332,  0.0217,  0.0316, -0.0825,\n",
              "          0.0558, -0.0924, -0.0390,  0.0579, -0.0364,  0.0042,  0.0487, -0.0918,\n",
              "          0.0615, -0.0629, -0.0648,  0.0839,  0.0521, -0.0418,  0.0191,  0.0073,\n",
              "          0.0643, -0.0222,  0.0030, -0.0429, -0.0326,  0.0848,  0.0495, -0.0583,\n",
              "          0.0192, -0.0639,  0.0128,  0.0565, -0.0341, -0.0213, -0.0286, -0.0196,\n",
              "          0.0852, -0.0991,  0.0622, -0.0967, -0.0746,  0.0024,  0.0010, -0.0672,\n",
              "         -0.0010, -0.0211, -0.0524, -0.0731,  0.0547, -0.0849,  0.0575, -0.0952,\n",
              "         -0.0381, -0.0276,  0.0583,  0.0457, -0.0191,  0.0456,  0.0828,  0.0367,\n",
              "         -0.0669,  0.0793, -0.0278, -0.0074]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "metadata": {
        "id": "zQkjsvH7BRuB",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Loss function**"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "q6WAiVnzulE7",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TqN5BDW-BWb5",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training and evaluation"
      ]
    },
    {
      "metadata": {
        "id": "Bud9NH2IDsvD",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "DONE:\n",
        "\n",
        "reduced learning capacity\n",
        "\n",
        "reduced learning rate\n",
        "\n",
        "doubled training and testing dataset\n",
        "\n",
        "TODO:\n",
        "\n",
        "keep tuning \n",
        "\n",
        "wasserstein for loss\n",
        "\n",
        "weighted cross-entropy:\n",
        "loss = nn.CrossEntropyLoss(weight = weight.type(dtypeFloat))(y,y_target)\n",
        "\n",
        "train loss okay, test loss fluctuating:  overfitting? try dropout/L1 L2 regularization on weights or activations"
      ]
    },
    {
      "metadata": {
        "id": "5ImFwAMNKpFc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        " **Training by image**"
      ]
    },
    {
      "metadata": {
        "id": "UrxsI3tth4sp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def eval_on_test_set():\n",
        "\n",
        "    running_loss=0\n",
        "    num_batches=0    \n",
        "    \n",
        "    correct = 0\n",
        "    \n",
        "    correct_0 = 0\n",
        "    correct_1 = 0\n",
        "    correct_2 = 0\n",
        "    correct_3 = 0\n",
        "    correct_4 = 0\n",
        "    \n",
        "    h = torch.zeros(1, 1, 40)\n",
        "    c = torch.zeros(1, 1, 40)\n",
        "   \n",
        "    h=h.to(device)\n",
        "    c=c.to(device)\n",
        "       \n",
        "    for count in range( len(test_data_amend)) :\n",
        "               \n",
        "        minibatch_data =  test_data_amend[ count  ]\n",
        "        minibatch_label = test_label_tensor[ count  ]\n",
        "        minibatch_label = torch.LongTensor([minibatch_label]).view(-1)\n",
        "        \n",
        "        minibatch_data=minibatch_data.to(device)\n",
        "        minibatch_label=minibatch_label.to(device)\n",
        "                                  \n",
        "        scores = net( minibatch_data, h , c)\n",
        "        \n",
        "        loss = criterion(  scores ,  minibatch_label )    \n",
        "        \n",
        "        h=h.detach()\n",
        "        c=c.detach()\n",
        "            \n",
        "        running_loss += loss.item()\n",
        "        num_batches += 1        \n",
        "\n",
        "        _, predicted = torch.max(scores,1)\n",
        "        #print(predicted,scores,minibatch_label)\n",
        "        #total += minibatch_label.size(0)\n",
        "        if predicted == minibatch_label:\n",
        "            correct+=1\n",
        "            if predicted == 0:\n",
        "                correct_0+=1\n",
        "            elif predicted == 1:\n",
        "                correct_1+=1\n",
        "            elif predicted == 2:\n",
        "                correct_2+=1                \n",
        "            elif predicted == 3:\n",
        "                correct_3+=1        \n",
        "            elif predicted == 4:\n",
        "                correct_4+=1                    \n",
        "        #print([predicted, minibatch_label[0], minibatch_label])\n",
        "    \n",
        "    total_loss = running_loss/num_batches \n",
        "    print('test: exp(loss) = ', math.exp(total_loss)  )\n",
        "    print ('correct', correct, 'correcr_0', correct_0,'correct_1',correct_1,'correcr_2', correct_2,'correct_3',correct_3,'correct_4',correct_4)\n",
        "    print ('Test accuracy:{}%'.format(100 * correct / len(test_data_amend)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "ApWfPoXvulE_",
        "outputId": "54941c68-7310-4a03-ff09-5ccbda29f1f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2985
        }
      },
      "cell_type": "code",
      "source": [
        "start=time.time()\n",
        "lr = 0.001 #default for adam\n",
        "\n",
        "for epoch in range(100):\n",
        " \n",
        "    if epoch > 5:\n",
        "        lr = lr / 2\n",
        "    \n",
        "    if lr < 0.0001:\n",
        "        lr = 0.0001\n",
        "        \n",
        "    optimizer=torch.optim.Adam( net.parameters(),lr = lr)\n",
        "        \n",
        "    running_loss=0\n",
        "    num_batches=0    \n",
        "       \n",
        "    h = torch.zeros( 1, 1, 40)\n",
        "    c = torch.zeros( 1, 1, 40)\n",
        "    \n",
        "    h=h.to(device)\n",
        "    c=c.to(device)\n",
        "    \n",
        "    for count in range( len(train_data_amend)):\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        minibatch_data = train_data_amend[ count]\n",
        "        minibatch_label = train_label_tensor[count]\n",
        "        minibatch_label = torch.LongTensor([minibatch_label]).view(-1)\n",
        "        #print (minibatch_data.shape)\n",
        "        #print (minibatch_label)\n",
        "\n",
        "        minibatch_data=minibatch_data.to(device)\n",
        "        minibatch_label=minibatch_label.to(device)\n",
        "        \n",
        "        h=h.detach()\n",
        "        c=c.detach()\n",
        "        h=h.requires_grad_()\n",
        "        c=c.requires_grad_()\n",
        "                       \n",
        "        # forward the minibatch through the net        \n",
        "        scores = net( minibatch_data, h , c )\n",
        "        \n",
        "        # Compute the average of the losses of the data points in this huge batch\n",
        "        loss = criterion(  scores ,  minibatch_label )\n",
        "        \n",
        "        # backward pass to compute dL/dR, dL/dV and dL/dW\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "                    \n",
        "        # update the running loss  \n",
        "        running_loss += loss.item()\n",
        "        num_batches += 1\n",
        "                \n",
        "    # compute stats for the full training set\n",
        "    total_loss = running_loss/num_batches\n",
        "    elapsed = time.time()-start\n",
        "    \n",
        "    print('')\n",
        "    print('epoch=',epoch, '\\t time=', elapsed, '\\t exp(loss)=',  math.exp(total_loss))\n",
        "    \n",
        "    eval_on_test_set()\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch= 0 \t time= 371.83787536621094 \t exp(loss)= 2.2207724025692888\n",
            "test: exp(loss) =  1.852122313788179\n",
            "correct 3135 correcr_0 576 correct_1 886 correcr_2 485 correct_3 915 correct_4 273\n",
            "Test accuracy:78.375%\n",
            "\n",
            "epoch= 1 \t time= 771.8687386512756 \t exp(loss)= 1.8060085159950163\n",
            "test: exp(loss) =  1.689108645482406\n",
            "correct 3280 correcr_0 622 correct_1 908 correcr_2 526 correct_3 937 correct_4 287\n",
            "Test accuracy:82.0%\n",
            "\n",
            "epoch= 2 \t time= 1225.363130569458 \t exp(loss)= 1.7392011626571986\n",
            "test: exp(loss) =  1.7134471482558267\n",
            "correct 3250 correcr_0 655 correct_1 898 correcr_2 505 correct_3 926 correct_4 266\n",
            "Test accuracy:81.25%\n",
            "\n",
            "epoch= 3 \t time= 1651.7655174732208 \t exp(loss)= 1.6953867877652469\n",
            "test: exp(loss) =  1.6634108130258214\n",
            "correct 3291 correcr_0 664 correct_1 890 correcr_2 526 correct_3 921 correct_4 290\n",
            "Test accuracy:82.275%\n",
            "\n",
            "epoch= 4 \t time= 2092.6755650043488 \t exp(loss)= 1.6652030207688964\n",
            "test: exp(loss) =  1.6392482907393584\n",
            "correct 3323 correcr_0 658 correct_1 921 correcr_2 546 correct_3 905 correct_4 293\n",
            "Test accuracy:83.075%\n",
            "\n",
            "epoch= 5 \t time= 2528.183450937271 \t exp(loss)= 1.6436653362026536\n",
            "test: exp(loss) =  1.7260662375773754\n",
            "correct 3243 correcr_0 541 correct_1 917 correcr_2 553 correct_3 944 correct_4 288\n",
            "Test accuracy:81.075%\n",
            "\n",
            "epoch= 6 \t time= 2966.4680473804474 \t exp(loss)= 1.5547479783365492\n",
            "test: exp(loss) =  1.540296754040085\n",
            "correct 3401 correcr_0 692 correct_1 915 correcr_2 527 correct_3 947 correct_4 320\n",
            "Test accuracy:85.025%\n",
            "\n",
            "epoch= 7 \t time= 3382.9185769557953 \t exp(loss)= 1.4836823591345183\n",
            "test: exp(loss) =  1.5044687071978045\n",
            "correct 3422 correcr_0 712 correct_1 912 correcr_2 537 correct_3 938 correct_4 323\n",
            "Test accuracy:85.55%\n",
            "\n",
            "epoch= 8 \t time= 3828.070792198181 \t exp(loss)= 1.4383259975410292\n",
            "test: exp(loss) =  1.4846484284897554\n",
            "correct 3442 correcr_0 737 correct_1 918 correcr_2 523 correct_3 932 correct_4 332\n",
            "Test accuracy:86.05%\n",
            "\n",
            "epoch= 9 \t time= 4258.950209140778 \t exp(loss)= 1.4233946735022822\n",
            "test: exp(loss) =  1.4713671441635499\n",
            "correct 3478 correcr_0 753 correct_1 924 correcr_2 520 correct_3 944 correct_4 337\n",
            "Test accuracy:86.95%\n",
            "\n",
            "epoch= 10 \t time= 4659.965521097183 \t exp(loss)= 1.4074693558839497\n",
            "test: exp(loss) =  1.467596830228095\n",
            "correct 3494 correcr_0 763 correct_1 921 correcr_2 522 correct_3 944 correct_4 344\n",
            "Test accuracy:87.35%\n",
            "\n",
            "epoch= 11 \t time= 5056.324677944183 \t exp(loss)= 1.395461811061707\n",
            "test: exp(loss) =  1.472543778799305\n",
            "correct 3486 correcr_0 750 correct_1 924 correcr_2 526 correct_3 951 correct_4 335\n",
            "Test accuracy:87.15%\n",
            "\n",
            "epoch= 12 \t time= 5451.8662095069885 \t exp(loss)= 1.3881914774843385\n",
            "test: exp(loss) =  1.4792839411631271\n",
            "correct 3460 correcr_0 736 correct_1 923 correcr_2 524 correct_3 938 correct_4 339\n",
            "Test accuracy:86.5%\n",
            "\n",
            "epoch= 13 \t time= 5851.356017589569 \t exp(loss)= 1.3830494236362498\n",
            "test: exp(loss) =  1.468963307790004\n",
            "correct 3476 correcr_0 753 correct_1 922 correcr_2 523 correct_3 940 correct_4 338\n",
            "Test accuracy:86.9%\n",
            "\n",
            "epoch= 14 \t time= 6211.236490011215 \t exp(loss)= 1.3749344662653924\n",
            "test: exp(loss) =  1.477019180693615\n",
            "correct 3479 correcr_0 750 correct_1 922 correcr_2 530 correct_3 945 correct_4 332\n",
            "Test accuracy:86.975%\n",
            "\n",
            "epoch= 15 \t time= 6572.878668546677 \t exp(loss)= 1.366265318904624\n",
            "test: exp(loss) =  1.4641040029804997\n",
            "correct 3475 correcr_0 758 correct_1 917 correcr_2 525 correct_3 943 correct_4 332\n",
            "Test accuracy:86.875%\n",
            "\n",
            "epoch= 16 \t time= 6933.989001750946 \t exp(loss)= 1.3553517144364147\n",
            "test: exp(loss) =  1.4635723792411397\n",
            "correct 3485 correcr_0 763 correct_1 920 correcr_2 515 correct_3 944 correct_4 343\n",
            "Test accuracy:87.125%\n",
            "\n",
            "epoch= 17 \t time= 7296.053114652634 \t exp(loss)= 1.3501982262255494\n",
            "test: exp(loss) =  1.4732952421689098\n",
            "correct 3485 correcr_0 758 correct_1 923 correcr_2 521 correct_3 944 correct_4 339\n",
            "Test accuracy:87.125%\n",
            "\n",
            "epoch= 18 \t time= 7658.003756046295 \t exp(loss)= 1.3408615852579022\n",
            "test: exp(loss) =  1.4685654256963054\n",
            "correct 3495 correcr_0 758 correct_1 922 correcr_2 520 correct_3 956 correct_4 339\n",
            "Test accuracy:87.375%\n",
            "\n",
            "epoch= 19 \t time= 8016.341433763504 \t exp(loss)= 1.3332051650089747\n",
            "test: exp(loss) =  1.4710556825380428\n",
            "correct 3496 correcr_0 757 correct_1 923 correcr_2 537 correct_3 944 correct_4 335\n",
            "Test accuracy:87.4%\n",
            "\n",
            "epoch= 20 \t time= 8403.130312204361 \t exp(loss)= 1.3280111341332068\n",
            "test: exp(loss) =  1.4755292725591627\n",
            "correct 3485 correcr_0 753 correct_1 917 correcr_2 529 correct_3 948 correct_4 338\n",
            "Test accuracy:87.125%\n",
            "\n",
            "epoch= 21 \t time= 8797.856326580048 \t exp(loss)= 1.3193974345503359\n",
            "test: exp(loss) =  1.479731575446849\n",
            "correct 3481 correcr_0 753 correct_1 913 correcr_2 533 correct_3 943 correct_4 339\n",
            "Test accuracy:87.025%\n",
            "\n",
            "epoch= 22 \t time= 9165.697852134705 \t exp(loss)= 1.3146231833113984\n",
            "test: exp(loss) =  1.4829213653196822\n",
            "correct 3480 correcr_0 749 correct_1 922 correcr_2 527 correct_3 942 correct_4 340\n",
            "Test accuracy:87.0%\n",
            "\n",
            "epoch= 23 \t time= 9528.139253377914 \t exp(loss)= 1.3064387434832834\n",
            "test: exp(loss) =  1.4764514948095575\n",
            "correct 3516 correcr_0 763 correct_1 920 correcr_2 541 correct_3 951 correct_4 341\n",
            "Test accuracy:87.9%\n",
            "\n",
            "epoch= 24 \t time= 9895.982473611832 \t exp(loss)= 1.3063432638218677\n",
            "test: exp(loss) =  1.4813505892517385\n",
            "correct 3489 correcr_0 758 correct_1 917 correcr_2 530 correct_3 948 correct_4 336\n",
            "Test accuracy:87.225%\n",
            "\n",
            "epoch= 25 \t time= 10257.263849496841 \t exp(loss)= 1.2935341556746731\n",
            "test: exp(loss) =  1.4802116540583425\n",
            "correct 3492 correcr_0 754 correct_1 915 correcr_2 529 correct_3 951 correct_4 343\n",
            "Test accuracy:87.3%\n",
            "\n",
            "epoch= 26 \t time= 10617.622632026672 \t exp(loss)= 1.2887331938289026\n",
            "test: exp(loss) =  1.493799460291\n",
            "correct 3483 correcr_0 762 correct_1 920 correcr_2 520 correct_3 943 correct_4 338\n",
            "Test accuracy:87.075%\n",
            "\n",
            "epoch= 27 \t time= 10985.111024856567 \t exp(loss)= 1.2879192130830797\n",
            "test: exp(loss) =  1.4810280369512678\n",
            "correct 3503 correcr_0 765 correct_1 919 correcr_2 523 correct_3 947 correct_4 349\n",
            "Test accuracy:87.575%\n",
            "\n",
            "epoch= 28 \t time= 11349.74022603035 \t exp(loss)= 1.28088550981567\n",
            "test: exp(loss) =  1.4839143460728932\n",
            "correct 3495 correcr_0 752 correct_1 919 correcr_2 529 correct_3 949 correct_4 346\n",
            "Test accuracy:87.375%\n",
            "\n",
            "epoch= 29 \t time= 11716.137433290482 \t exp(loss)= 1.2730987257378008\n",
            "test: exp(loss) =  1.495070411793796\n",
            "correct 3501 correcr_0 749 correct_1 930 correcr_2 528 correct_3 946 correct_4 348\n",
            "Test accuracy:87.525%\n",
            "\n",
            "epoch= 30 \t time= 12079.921563148499 \t exp(loss)= 1.2690881647540777\n",
            "test: exp(loss) =  1.4954294277992786\n",
            "correct 3495 correcr_0 759 correct_1 924 correcr_2 530 correct_3 945 correct_4 337\n",
            "Test accuracy:87.375%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-51f8f4ee0c85>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;31m# update the running loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     98\u001b[0m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mbias_correction1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}