08. Performance Measure.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from pycocotools.coco import COCO\n",
    "from pycocoevalcap.eval import COCOEvalCap\n",
    "import matplotlib.pyplot as plt\n",
    "import skimage.io as io\n",
    "import pylab\n",
    "pylab.rcParams['figure.figsize'] = (10.0, 8.0)\n",
    "\n",
    "import json\n",
    "from json import encoder\n",
    "encoder.FLOAT_REPR = lambda o: format(o, '.3f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up file names and pathes\n",
    "dataDir = 'coco'\n",
    "dataType = 'val2014'\n",
    "algName = 'beam_search_k5'\n",
    "annFile = '%s/annotations/captions_%s.json'%(dataDir,dataType)\n",
    "# annFile = 'captions_coco_val2014_results.json'\n",
    "subtypes = ['results', 'evalImgs', 'eval']\n",
    "# [resFile, evalImgsFile, evalFile]= ['%s/results/captions_%s_%s_%s.json'%(dataDir,dataType,algName,subtype) for subtype in subtypes]\n",
    "resFile = '{}/results/captions_coco_val2014_results.json'.format(dataDir)\n",
    "# download Stanford models\n",
    "# !./get_stanford_models.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.33s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.02s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    }
   ],
   "source": [
    "# create coco object and cocoRes object\n",
    "coco = COCO(annFile)\n",
    "cocoRes = coco.loadRes(resFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenization...\n",
      "setting up scorers...\n",
      "computing Bleu score...\n",
      "{'testlen': 55234, 'reflen': 52811, 'guess': [55234, 50171, 45108, 40045], 'correct': [32570, 14575, 6607, 3030]}\n",
      "ratio: 1.0458805930582447\n",
      "Bleu_1: 0.590\n",
      "Bleu_2: 0.414\n",
      "Bleu_3: 0.293\n",
      "Bleu_4: 0.209\n",
      "computing METEOR score...\n",
      "METEOR: 0.218\n",
      "computing Rouge score...\n",
      "ROUGE_L: 0.463\n",
      "computing CIDEr score...\n",
      "CIDEr: 0.695\n",
      "computing SPICE score...\n",
      "SPICE: 0.148\n"
     ]
    }
   ],
   "source": [
    "# create cocoEval object by taking coco and cocoRes\n",
    "cocoEval = COCOEvalCap(coco, cocoRes)\n",
    "\n",
    "# evaluate on a subset of images by setting\n",
    "# cocoEval.params['image_id'] = cocoRes.getImgIds()\n",
    "# please remove this line when evaluating the full validation set\n",
    "cocoEval.params['image_id'] = cocoRes.getImgIds()\n",
    "\n",
    "# evaluate results\n",
    "# SPICE will take a few minutes the first time, but speeds up due to caching\n",
    "cocoEval.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bleu_1: 0.590\n",
      "Bleu_2: 0.414\n",
      "Bleu_3: 0.293\n",
      "Bleu_4: 0.209\n",
      "METEOR: 0.218\n",
      "ROUGE_L: 0.463\n",
      "CIDEr: 0.695\n",
      "SPICE: 0.148\n"
     ]
    }
   ],
   "source": [
    "# print output evaluation scores\n",
    "for metric, score in cocoEval.eval.items():\n",
    "    print('%s: %.3f'%(metric, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ground truth captions\n",
      "A man with a red helmet on a small moped on a dirt road. \n",
      "Man riding a motor bike on a dirt road on the countryside.\n",
      "A man riding on the back of a motorcycle.\n",
      "A dirt path with a young person on a motor bike rests to the foreground of a verdant area with a bridge and a background of cloud-wreathed mountains. \n",
      "A man in a red shirt and a red hat is on a motorcycle on a hill side.\n",
      "\n",
      "\n",
      "generated caption (CIDEr score 0.0)\n",
      "a group of people standing on top of a lush green field\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "No such file: '/mnt/d/Projects/mscoco/coco/images/val2014/COCO_val2014_000000391895.jpg'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-27012388037f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoco\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadImgs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimgId\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mI\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%s/images/%s/%s'\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataDir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdataType\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'file_name'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mI\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'off'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/skimage/io/_io.py\u001b[0m in \u001b[0;36mimread\u001b[0;34m(fname, as_gray, plugin, **plugin_args)\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mfile_or_url_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'imread'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mplugin_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ndim'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/skimage/io/manage_plugins.py\u001b[0m in \u001b[0;36mcall_plugin\u001b[0;34m(kind, *args, **kwargs)\u001b[0m\n\u001b[1;32m    208\u001b[0m                                (plugin, kind))\n\u001b[1;32m    209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/skimage/io/_plugins/imageio_plugin.py\u001b[0m in \u001b[0;36mimread\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimageio_imread\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimageio_imread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/imageio/core/functions.py\u001b[0m in \u001b[0;36mimread\u001b[0;34m(uri, format, **kwargs)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m     \u001b[0;31m# Get reader and read first\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m     \u001b[0mreader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"i\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/imageio/core/functions.py\u001b[0m in \u001b[0;36mget_reader\u001b[0;34m(uri, format, mode, **kwargs)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;31m# Create request object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m     \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;31m# Get format\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/imageio/core/request.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, uri, mode, **kwargs)\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# Parse what was given\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parse_uri\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muri\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;31m# Set extension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/d/Projects/mscoco/venv/lib/python3.6/site-packages/imageio/core/request.py\u001b[0m in \u001b[0;36m_parse_uri\u001b[0;34m(self, uri)\u001b[0m\n\u001b[1;32m    255\u001b[0m                 \u001b[0;31m# Reading: check that the file exists (but is allowed a dir)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mFileNotFoundError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No such file: '%s'\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m                 \u001b[0;31m# Writing: check that the directory to write to does exist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: No such file: '/mnt/d/Projects/mscoco/coco/images/val2014/COCO_val2014_000000391895.jpg'"
     ]
    }
   ],
   "source": [
    "# demo how to use evalImgs to retrieve low score result\n",
    "evals = [eva for eva in cocoEval.evalImgs if eva['CIDEr']<30]\n",
    "print('ground truth captions')\n",
    "imgId = evals[0]['image_id']\n",
    "annIds = coco.getAnnIds(imgIds=imgId)\n",
    "anns = coco.loadAnns(annIds)\n",
    "coco.showAnns(anns)\n",
    "\n",
    "print('\\n')\n",
    "print('generated caption (CIDEr score %0.1f)'%(evals[0]['CIDEr']))\n",
    "annIds = cocoRes.getAnnIds(imgIds=imgId)\n",
    "anns = cocoRes.loadAnns(annIds)\n",
    "coco.showAnns(anns)\n",
    "\n",
    "img = coco.loadImgs(imgId)[0]\n",
    "I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name']))\n",
    "plt.imshow(I)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnMAAAH+CAYAAAD3doiVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de/yt9Zz//8dTB6VQKUnt2iHHQcwWM6U2kZKvYoaRQ2XMhJHJjBniaxQN4uvsh6+QQqTfhKIc0kEaOuxS6SBt2qmk83Eiqtf3j+v6aFmtz2evtff6fNa+9n7cb7frttZ6X+/rul7rsPs8uw7vK1WFJEmSuul+ky5AkiRJy84wJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJixJJTll0nVoOEnWSPKuJJcmubP9/nafdF2SVl2GOWkM2j/oMw7amGRJ22/+GLc7v13nYeNap5bqzcA7gd8AHwTeBfx8mAWTrJPkTUlOSnJtkj8kuTnJmUnek+QRff0Pa7/fvadpn5ruTnJLkl8m+WaSfZM8ZJoaFvYtO3Balg9mhvf9kiTfbd/zH5PckOSiJF9Ostc4tyWtilafdAGSeBxwx6SL0NBeANwOPLeq/jDsQkmeAfwXsClwJXA8TSBcB3gK8Fbg35M8o6rOGXK1xwDnts8fCMwDngnsBrwnyX5Vddg0y14OTDdvbJIcAvwj8DvgOOAyIMBjgf8FLAQOn+06pJWZYU6asKoaaq+OVhgPB24YMcg9FvgesC6wP/Chqrqrr8+WwPuBB41Qyzf7w1qS1YG/Bz4GfCHJnVX11QHLLqmqA0fY1siSbEcT5K4E/qqqruybvwZNmJO0HDzMKk3YoHPmkjwwyX8kuSDJrUluaw+hfS3JX7Z9DqTZywGwV99hsr171nW/JK9LclaS25P8T/v89UkG/jcgySuSnJPkd+2hsS8leXiSU/oPwfUctjswyTZJjktyY+8h5STPSnJIe2jt1na9FyQ5IMlaA7Z/YLv8wiR7JDk7yR1JfpPkw0nu3/Z7dlvTrUluausceHhxhs//wUnel+SSJL9v1/O9JM/p63dY+963BLbo+ayXDLGZT9CEtPdX1fv7gxxAVV1WVS8FfjJK/QPWc1dVHQL8U9v04SRrL886p773JGsmeWf7Wd2ZpR/e/+v28ej+INfW+seqOmGabe6U5Fvt7+/OJFckOWbA9zLS73vq31uShyX5XJKr0hym3runzwOSvC3Jue36bk/ykyR7DFhfkuyV5MdJrmt/Q1e0v6G/W8rnI42Fe+akFUySAN+l+UP4E+BzwF3AZsCzgB8BZwOnAOsB+wHnAd/sWc25Pc+/BLwcuKJdVwEvAj4FbAe8om/7b6HZQ3QTzeGvW4DnAv/dPp/OXwFvA04DDgU2BKb2Xr2V5rDaj2kOta0FbAscCCxM8pyqunvAOt8I7NK+t1OAnYB/ATZIcgxwZLu+Q9rP65XtdneZoc7e97pe+74eD5wFfLRd/qXA95O8vqo+03b/JrAEeFP7+qPt481L2caWwHOA3wMfWFpNVXXnMLUP4XDgAGAL4Nk0n9PyOhp4GvAdms/j2qX0v6F93GqUjSR5F815ibe327mCZo/o1Hf8g57uI/2+WxsAp7fr/zpwD3BNu+31gJNoDn2fQ/Nbvh/wPOArSZ5QVe/oWdd7aH73lwFH0fwb2YTmc3oJ8LVR3ru0TKrKyclpOSeaPyBFE06mm25u+8wfsOwpPa+f2LZ9Y8B27ges3/N6ftv3sGnq2qOdfw6wbk/7OsCidt7Le9ofAfwRuA6Y19Me4KtT77NvGwt73v9rp6njEUAGtB/ULvd3fe0Htu23AI/rab8/cCFwN01Q2KHvszmhXW7rIb+3z7T9P9NbH034uAW4c8D3tYTmEOWwv41Xtds4bRl+V4e1y+49TPuA5b/U9nvXgO9ryQy/1Zf1reeUdpnzgQ1HqH/Tnt/9sTSha6tBv4WeZXZq+/8K2HTA/M2W9ffd92/1i8DqM3zmb+lrX4vmf7Lu6f19tb/DK4EHDFjX0J+Vk9PyTBMvwMlpZZh6/kAMM80fsOwpPa+nwtxXhtjufGYOc1PhZqcB83Zs553U0/aOtu2dA/pvQbOHsPrap8LBT5fhc9ugXfbQvvYD2/aDBizzzqk/xgPm7dXO22uIba8J/A9wG7DBgPlTQfOdfe1LGC3MvaVdz5HL8PlMBYu9h2kfsPzBbb9PDfi+Zpq+2beeU9r23ZbhPTwLWNy3/lvbYPRKYLW+/t9q+7xoiHWP9Ptu24smpD90wDIPaX/jZ02zvSe3y3+gp+0Gmr1y9x/1s3FyGtfkYVZpjKoq081rz63aYojVXERzmHSPJFvQXLF4GrCoRjjpvvVUmj0JpwyY90OaPVxP6Wmben5af+equjzJFTQBcpAzpysiyTo0h4NfBDya5srL3s9q02kWXTSg7Tft49kD5l3VPm42XS09HgM8APjvqrpxwPyTaMLtUwbM64qpz7gGzPthVS0ccX3TfsfTqaqTkzya5rD6DjSf57Y0hy2fR3O+5wvq3sPLz2jr/e4Qqx/19z1lSVUNOkT8NGA1oNKck9pvjfbxcT1tR9CcDnBRkqPa7f6kqmY6JUEaK8OctIKpqruTPJtmD9Tf0py/BnBbksOBt1XV7UOu7sHAjYNCYFXdleR64KF9/aE9f2iAa5g+zP12UGOaKxZPArYBLqA5h+g6msO50JzXdf9p1jnoD+JdQ8xbY8C8flPv9epp5k+1rzfEumYytZ7pAutsenj7eN2Y1jfwO16aqrqH5lzPH8Gfzgt9Ls15fc8BXs+95yCuB9xUVb8bYtWj/r6nTPc+pi6eeVo7TWfdnuf/QnNI+NU0VyrvD9yV5HjgzVW1eOa3IC0/r2aVVkBVdVNV/UtVzaM5x+gfaAam3Rf49AiruoXmYoH7hJs0Q1hsSHPIa8rU842nWd907TB47w80Y55tQ3Mo+IlVtU9V/e9qhsX4zDTLzIWpMPiwaeZv0tdvWU3t5VyQ5MEz9hyj9krO7duXZ4xjnVU13Xc88nqq6vs0ez6huUBjys3A+kNegTvq7/tPJcywPoCPVFVmmJ7V817urqqPVtWTaf59/A3wDeCFwHenrryWZpNhTlrBVdXiqvo8zSGq22nC0ZSpK0BXm2bxn9L8O99+wLzt2+XO6esPzVWAf6Y95Dtv+Mr/5FHt49cHzNthGdY3LpfQDNb85PYKxn5Tf7CHHcB3oKq6jObqy7WAf19a/zH+8d8b2Jxmz+DJY1rnuN3WPvYecj+9fb3zEMuP+vtemjNpDts+c4Rl/qSqrq2qr1czxMxJwCOBv1iWdUmjMMxJK5gkW6bvtk6t9WkOR/YefrqJZi/D5tOs7tD28X1JHtCzjQfQnBwP8Pme/l+hOVT5xiTzevoHeB/Th8aZLGkfF/Y2tu/x/f2d50p7aO4ImvP3Duqdl+SRwD/THAr+0hg29880e4jeluTN7V6jP5Nk8yRH0gzxssySrJ7kH4FP0vw2/qWqfr8861yOWnZO8uJp9pyty73DvJzaM+sT7eOHktzn0HRf26i/7xm159EdQbMX9T+S3Of3nuSR7XAzJLl/km0H9FmD5uIe8O4umgOeMyeteJ4MfD3JWcDFNCf8b0SzR24NegJQVd2e5AzgmUmOAH5Bs7fu2Ko6v6q+kmQ3mnHTLkzyTZo/8LvTDH77tao6omd9v0zyTuC9wHlJvsa948xtQDOe3ZNGfD/forma8V+TPJFmb8rmNLfFOo7pg+hc2J9mL8y+SZ5Gswdrapy5BwL7tnvWlktVXZzkeTS38/ogsF+SE7n3dl5PprkooBgt4O6ee+/1uw7NZ/lMmkPEtwD7VNV045zNn+Yk/ykfraoZx9AbwmOBjwA3JfkRcCn3jpm4K835cWcA/9/UAlX1/ST/SXMI9uL2N3sFzSHM7Wj23O3d9h3p9z2kfWlObXg38Kokp9GcK/pwmgsfnkYzJMplwNrAaUkW01yQcznNHtjntn2PraqLR9y+NLpJX07r5LQyTAwYf21AnyUMNzTJZjRh6r9pTtS+k2Ycq+8AuwxY76NoAtMNNIeI/mzICpo98P9Ec2XoHe10NvAG4H7T1PoqmtD1e5qT579M88fsAuDmvr4L220eOMN7n0ezx+Mqmj2LF9IM2bF6//tv+x/Yti8csK69+9/jKLUMWGY9mgB1aftZ30wz5MV9hrvo+R6XLOPvZF2aE+ZPphlw9480oetsmj2fW/b1P2zQe+1pn5ruptnz90uaQXb3ZcBwK32f0dBD6NAOTbIM73dDmluLfZXmKu2buHccw5Pb3+Wa0yz7fJorWm9sv5craM5Fe3Zfv5F+34N+bwP6rNl+hj/m3vEGfw2cSLM38SFtvzXa3/F32vlT/15OB1433Xtzchr3lKqxnM8qaSWX5EE0eyjOrarlOhQoSRofz5mT9GeSbNR/jlN7jteHaA4hfWMihUmSBnLPnKQ/k+R1NOcL/YDm0NYGNFcGPppmMOO/ruHGAJMkzQEvgJDU7wyasdG2595BVC+juaH4+w1ykrRicc+cJElSh3nOnCRJUoetsodZN9xww5o/f/6ky5AkSVqqs88++/qq2mjQvFU2zM2fP59FixZNugxJkqSlSnL5dPM8zCpJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUodNNMwlmZfk5CQXJbkwyX5t+4FJrkpybjs9v2eZtyVZnOSSJM/rad+5bVucZP9JvB9JkqS5NulBg+8C3lxV5yR5IHB2khPaeR+pqg/2dk7yeOBlwBOAhwM/SPLodvYngecCVwJnJTm2qi6ak3chSZI0IRMNc1V1NXB1+/y2JBcDm86wyG7AkVV1J3BZksXANu28xVX1K4AkR7Z9DXOSJGmltsKcM5dkPvAU4Iy2ad8k5yc5NMn6bdumwBU9i13Ztk3XLkmStFJbIcJcknWBo4E3VdWtwKeBRwJb0+y5+9CYtrNPkkVJFl133XXjWKUkSdJETTzMJVmDJsgdUVVfB6iqa6rq7qq6B/gs9x5KvQqY17P4Zm3bdO1/pqoOqaoFVbVgo402Gv+bkSRJmmOTvpo1wOeBi6vqwz3tm/R0exFwQfv8WOBlSe6fZEtgK+BM4CxgqyRbJlmT5iKJY+fiPUiSJE3SpK9m3RZ4FfCzJOe2bW8H9kiyNVDAEuC1AFV1YZKjaC5suAt4Q1XdDZBkX+B7wGrAoVV14Vy+EUmSpElIVU26holYsGBBLVq0aNJlSJIkLVWSs6tqwaB5Ez9nTpIkScvOMCdJktRhhjlJkqQOm/QFECu9+fsfN+kSxmbJwbtOugRJktTHPXOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHXYRMNcknlJTk5yUZILk+zXtm+Q5IQkl7aP67ftSfLxJIuTnJ/kqT3r2qvtf2mSvSb1niRJkubSpPfM3QW8uaoeDzwDeEOSxwP7AydW1VbAie1rgF2ArdppH+DT0IQ/4ADg6cA2wAFTAVCSJGllNtEwV1VXV9U57fPbgIuBTYHdgMPbbocDu7fPdwO+WI3TgfWSbAI8Dzihqm6sqpuAE4Cd5/CtSJIkTcSk98z9SZL5wFOAM4CNq+rqdtZvgY3b55sCV/QsdmXbNl27JEnSSm2FCHNJ1gWOBt5UVbf2zquqAmpM29knyaIki6677rpxrFKSJGmiJh7mkqxBE+SOqKqvt83XtIdPaR+vbduvAub1LL5Z2zZd+5+pqkOqakFVLdhoo43G+0YkSZImYNJXswb4PHBxVX24Z9axwNQVqXsBx/S079le1foM4Jb2cOz3gJ2SrN9e+LBT2yZJkrRSW33C298WeBXwsyTntm1vBw4GjkryGuBy4KXtvOOB5wOLgTuAVwNU1Y1JDgLOavu9u6punJu3IEmSNDkTDXNVdRqQaWbvOKB/AW+YZl2HAoeOrzpJkqQV38TPmZMkSdKyM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHbb6sB2TrAbcv6ru6Gt/NrAbcAdwSFVdNt4SJUmSNJ1R9sx9ELgxyYOnGpK8DDgBeCPwVuDMJPPGW6IkSZKmM0qY2x44uapu6Wk7ALgZ2BN4C7Ae8K/jK0+SJEkzGSXMzQMWT71I8gjgMcAnqurLVfVB4DvAzuMtUZIkSdMZJcw9CLi15/W2QAHf7Wm7ENhsDHVJkiRpCKOEuauBLXtePwf4HXB2T9u6wF1jqEuSJElDGPpqVuB04IVJXgD8Hvhb4MSq+mNPny2Bq8ZYnyRJkmYwyp6597b9jwG+B6wJvGdqZpK1gGcCZ4yzQEmSJE1v6D1zVfWzJE8H9mqbvlZVZ/V0eQpwEvDVMdYnSZKkGYxymJWq+hnwb9PM+wnwonEUJUmSpOEMfZg1yUlJ9lxKn1cmOWn5y5IkSdIwRjlnbiEwfyl9tgB2WNZiJEmSNJpRwtww1sahSSRJkubMSOfM0QwSfB9JAmwOPB+4YnmLkiRJ0nBm3DOX5J4kdye5u206cOp170SzN+5XwNbAkbNcsyRJklpL2zN3Kvfujdse+DWwZEC/u4EbgBOBz42rOEmSJM1sxjBXVQunnie5B/hCVb17touSJEnScEY5Z25L4ObZKkSSJEmjG+UOEJfPZiGSJEka3UhXsyZZA9gN2AZYH1htQLeqqteMoTZJkiQtxdBhLsnDgROAxwKZoWsBhjlJkqQ5MMqeuQ8BjwO+CnyWZjw5BwiWJEmaoFHC3E7AqVX1itkqRpIkSaMZJcytBZwxW4VoxTd//+MmXcJYLDl410mXIEnS2Ixyb9YLgC3GufEkhya5NskFPW0HJrkqybnt9PyeeW9LsjjJJUme19O+c9u2OMn+46xRkiRpRTZKmPs/wAuTPH6M2z8M2HlA+0eqaut2Oh6g3e7LgCe0y3wqyWpJVgM+CewCPB7YY8w1SpIkrbBGOcx6LfAt4MdJPgaczTSDCFfVqcOssKpOTTJ/yO3vBhxZVXcClyVZTDNECsDiqvoVQJIj274XDbleSZKkzholzJ1CM+xIgP/g3nu2DjJo/LlR7JtkT2AR8OaqugnYFDi9p8+VbRs0V9b2tj990EqT7APsA7D55psvZ4mSJEmTN0qYezczB7hx+TRwULutg2iGRPn7cay4qg4BDgFYsGDBXLwXSZKkWTXK7bwOnMU6erdzzdTzJJ8Fvt2+vAqY19N1s7aNGdolSZJWaqNcADEnkmzS8/JFNFfRAhwLvCzJ/ZNsCWwFnAmcBWyVZMska9JcJHHsXNYsSZI0KSPdm3XcknwVWAhsmORK4ABgYZKtaQ6zLgFeC1BVFyY5iubChruAN1TV3e169gW+R3Ou3qFVdeEcvxVJkqSJGOXerCcN2bWqaschO+4xoPnzM/R/D/CeAe3HA8cPWZ8kSdJKY5Q9cwuXMn/qSlcvLJAkSZojQ58zV1X3GzQB69Pct/Vc4GvAmrNUqyRJkvos9wUQVXVLVf0AeC6wA/Dm5a5KkiRJQxnb1axVdSPNeWv/MK51SpIkaWbjHprkVsBbK0iSJM2RsYW5JGsDu9Lcw1WSJElzYJShSfacYR3zgJcDjwI+OIa6JEmSNIRRhiY5jMHDjqR9vAf4MvCO5axJkiRJQxolzL16mvZ7gJuARVX12+UvSZIkScMaOsxV1eGzWYgkSZJGN+6rWSVJkjSHRjnMCkCSzYE9gacA6wG3AOcAX6qqy8dbniRJkmYyUphL8o/Ax2lu2ZWeWbsD70iyX1V9Zoz1SZIkaQZDH2ZNsiPwf4E7gfcAzwYe1z7+J/B74JNtP0mSJM2BUfbM/TtwG/CXVfXLnvZLgFOSHA6c3fY7cXwlSpIkaTqjXACxDXBUX5D7k7b9/2/7SZIkaQ6MEubWBq5fSp/r2n6SJEmaA6OEuctpzo+bybOAXy97OZIkSRrFKGHuG8DTknwqyXq9M5I8KMnHaA6xfn2cBUqSJGl6o1wA8T7ghcDrgFckOQ+4GngY8GTgQcDP236SJEmaA0PvmauqW4G/Bj4LrAZsB7wEeCZNKPwssG3bT5IkSXNgpEGDq+oW4LVJ9gUeAzyY5g4Ql1TVH2ehPkmSJM1g5Nt5AbTB7YIx1yJJkqQRjXIHiEcm2TPJQ6aZv2E7/xHjK0+SJEkzGeVq1v2BDwHTnRN3C/BBmjtASJIkaQ6MEuYWAj+Y7ty4tv0Elj4WnSRJksZklDC3KbBkKX1+DTx8mauRJEnSSEYJc3+gGUtuJg8EatnLkSRJ0ihGCXMXALsmWWPQzCRrAi8ALhpHYZIkSVq6UcLcl4HNgaOSPKx3Rvv6KGAe8MXxlSdJkqSZjDLO3CHAi4HdgOcmOR+4iuZcuicBDwB+APzfcRcpSZKkwUa5ndc9wK7AwcAfgWcAf9M+/gF4L7Br20+SJElzYNTbef0ReHuSdwCPBdYDbgZ+boiTJEmae8t6O6978EIHSZKkiRvlAghJkiStYAxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhQ4e5JCcl2XMpfV6Z5KTlL0uSJEnDGGXP3EJg/lL6bAHssKzFSJIkaTTjPsy6NnDXmNcpSZKkaYx6B4ga1JgkwObA84ErlrcoSZIkDWfGPXNJ7klyd5K726YDp173TjR7434FbA0cOcs1S5IkqbW0PXOncu/euO2BXwNLBvS7G7gBOBH43LiKkyRJ0sxmDHNVtXDqeZJ7gC9U1btnuyhJkiQNZ5Rz5rYEbp6tQiRJkjS6ocNcVV0+m4VIkiRpdNOGuSTvXMZ1VlUdtIzLSpIkaQQz7Zk7cBnXWYBhTpIkaQ7MFOaeNWdVSJIkaZlMG+aq6odzWYgkSZJGN+7beUmSJGkOGeYkSZI6bOihSdpBgwfem7VPVdWo93yVJEnSMhgldPXe2qvXesCjgbWB83BgYUmSpDkzyqDBC6ebl+SBwEeAvwZevPxlSZIkaRhjOWeuqm4D9gHuAt4zjnVKkiRp6cZ2AURV3QOcDOw+rnVKkiRpZuO+mnUtYP0xr1OSJEnTGFuYS/JY4CXA4nGtU5IkSTMbZWiSQ2dYxzxgW2A14M1jqEuSJElDGGVokr2XMv/nwP+pqi8sezmSJEkaxShhbstp2u8Bbqqq28dQjyRJkkYwyjhzl89mIZIkSRrdWC6ASLJ+knXGsS5JkiQNb+gwl2THJB9Isn5P20OT/BC4HrgxyYdno0hJkiQNNsqeuTcCL66qm3raPgg8E/glcAOwX5KXjrE+SZIkzWCUMPdk4LSpF0nWBv4WOKGqHg08BrgCeN1YK5QkSdK0RglzDwV+0/P66TR3fDgM/nR/1m/ThDpJkiTNgVHC3J3A2j2vnwkUcGpP263ABmOoS5IkSUMYJcxdBjy75/XfAJdW1VU9bfNoLoaQJEnSHBglzB0OPDHJGUl+BDwR+EpfnycBl4yrOEmSJM1slDD3aeBIYAHNfVi/Dbx/amaSv6AJeKeMsT5JkiTNYOgwV1V/rKqXA+sDD66q3arqzp4uvwWeAnxi2HUmOTTJtUku6GnbIMkJSS5tH9dv25Pk40kWJzk/yVN7ltmr7X9pkr2G3b4kSVLXjXwHiKq6tb1ytb/9+qo6r6puGWF1hwE797XtD5xYVVsBJ7avAXYBtmqnfWj2FJJkA+AAmqtrtwEO6B3YWJIkaWU2cphLslGS1yX5WJLP9bVv044/N5SqOhW4sa95N5rz82gfd+9p/2I1TgfWS7IJ8Dyase5ubAc0PoH7BkRJkqSV0uqjdE7yGuDjNOPLhWZokn9oZ28M/IRmr9nnl6Omjavq6vb5b9v1AmxKMyjxlCvbtunaJUmSVnqj3Jv1ucAhwC+AF9Ee5pxSVRcAF3LvnrTlVlVFExjHIsk+SRYlWXTdddeNa7WSJEkTM8ph1rcCVwM7VNWxwLUD+pwPPH45a7qmPXxK+zi1natoxrGbslnbNl37fVTVIVW1oKoWbLTRRstZpiRJ0uSNEuYWAN+uqltn6HMl8LDlK4ljgakrUvcCjulp37O9qvUZwC3t4djvATslWb+98GGntk2SJGmlN8o5c2sC/7OUPusBdw+7wiRfBRYCGya5kuaq1IOBo9rz8y4HXtp2Px54PrAYuAN4NUBV3ZjkIOCstt+7q5RxMesAABAISURBVKr/ogpJkqSV0ihhbgnwl0vp83RGuANEVe0xzawdB/Qt4A3TrOdQ4NBhtytJkrSyGOUw6zHAM5O8ZNDMJK+muZ3X0eMoTJIkSUs3yp65DwAvA76a5G+BBwMk2Rd4JvBi4FJGuAOEJEmSls/QYa6qbkqyA/BFoHfv3Mfbxx8BL6+qpZ1XJ0mSpDEZadDgqvo1sDDJk4C/Ah4C3AKcXlVnz0J9kiRJmsHQYS7J9sCtVXVuVZ1PM6acJEmSJmiUCyBOprlVlyRJklYQo4S564HfzVYhkiRJGt0oYe4U4K9nqQ5JkiQtg1HC3DuAxyQ5KMkas1WQJEmShjfK1axvAy4A3g68Jsl5wG+B6utXVfWaMdUnSZKkGYwS5vbuef6wdhqkAMOcJEnSHBglzG05a1VIkiRpmYxyB4jLZ7MQSZIkjW6UCyAkSZK0gjHMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYYY5SZKkDjPMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYYY5SZKkDjPMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYYY5SZKkDjPMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYYY5SZKkDjPMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYYY5SZKkDlt90gVIc23+/sdNuoSxWXLwrpMuQZI0Ye6ZkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOW2HDXJIlSX6W5Nwki9q2DZKckOTS9nH9tj1JPp5kcZLzkzx1stVLkiTNjRU2zLWeVVVbV9WC9vX+wIlVtRVwYvsaYBdgq3baB/j0nFcqSZI0ASt6mOu3G3B4+/xwYPee9i9W43RgvSSbTKJASZKkubQih7kCvp/k7CT7tG0bV9XV7fPfAhu3zzcFruhZ9sq27c8k2SfJoiSLrrvuutmqW5Ikac6sPukCZrBdVV2V5KHACUl+3juzqipJjbLCqjoEOARgwYIFIy0rSZK0Ilph98xV1VXt47XAN4BtgGumDp+2j9e23a8C5vUsvlnbJkmStFJbIcNcknWSPHDqObATcAFwLLBX220v4Jj2+bHAnu1Vrc8Abuk5HCtJkrTSWlEPs24MfCMJNDV+paq+m+Qs4KgkrwEuB17a9j8eeD6wGLgDePXclyxJkjT3VsgwV1W/Ap48oP0GYMcB7QW8YQ5KkyRJWqGskIdZJUmSNBzDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHbb6pAuQtOzm73/cpEsYmyUH7zrpEiSpk9wzJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDDHOSJEkdZpiTJEnqMMOcJElShxnmJEmSOswwJ0mS1GGGOUmSpA4zzEmSJHWYYU6SJKnDVp90AZIEMH//4yZdwlgsOXjXSZcgaRXjnjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJkqQOM8xJkiR1mGFOkiSpwwxzkiRJHbb6pAuQpJXJ/P2Pm3QJY7Pk4F0nXYKkIaxUe+aS7JzkkiSLk+w/6XokSZJm20qzZy7JasAngecCVwJnJTm2qi6abGWS1E3uZZS6YWXaM7cNsLiqflVVfwCOBHabcE2SJEmzaqXZMwdsClzR8/pK4OkTqkWStAJxL+OKx+9kfFamMLdUSfYB9mlf3p7kkjnY7IbA9XOwHf05P/fJ8bOfHD/7yZjTzz3vn6stdcIK8Zufo+9ki+lmrExh7ipgXs/rzdq2P6mqQ4BD5rKoJIuqasFcblN+7pPkZz85fvaT4ec+OX72jZXpnLmzgK2SbJlkTeBlwLETrkmSJGlWrTR75qrqriT7At8DVgMOraoLJ1yWJEnSrFppwhxAVR0PHD/pOvrM6WFd/Ymf++T42U+On/1k+LlPjp89kKqadA2SJElaRivTOXOSJEmrHMPcLPHWYpOR5NAk1ya5YNK1rGqSzEtycpKLklyYZL9J17QqSLJWkjOTnNd+7u+adE2rmiSrJflpkm9PupZVSZIlSX6W5NwkiyZdzyR5mHUWtLcW+wU9txYD9vDWYrMvyfbA7cAXq+ovJl3PqiTJJsAmVXVOkgcCZwO7+7ufXUkCrFNVtydZAzgN2K+qTp9waauMJP8KLAAeVFUvmHQ9q4okS4AFVTXxceYmzT1zs8Nbi01IVZ0K3DjpOlZFVXV1VZ3TPr8NuJjmziyaRdW4vX25Rjv5f+lzJMlmwK7A5yZdi1ZdhrnZMejWYv5R0yojyXzgKcAZk61k1dAe5jsXuBY4oar83OfOR4G3APdMupBVUAHfT3J2e4enVZZhTtJYJVkXOBp4U1XdOul6VgVVdXdVbU1z55ttkniKwRxI8gLg2qo6e9K1rKK2q6qnArsAb2hPs1klGeZmx1JvLSatjNpzto4Gjqiqr0+6nlVNVd0MnAzsPOlaVhHbAi9sz906Enh2ki9PtqRVR1Vd1T5eC3yD5hSnVZJhbnZ4azGtctoT8T8PXFxVH550PauKJBslWa99vjbNhVc/n2xVq4aqeltVbVZV82n+O39SVb1ywmWtEpKs015oRZJ1gJ2AVXYUA8PcLKiqu4CpW4tdDBzlrcXmRpKvAj8BHpPkyiSvmXRNq5BtgVfR7J04t52eP+miVgGbACcnOZ/mfyRPqCqHyNDKbmPgtCTnAWcCx1XVdydc08Q4NIkkSVKHuWdOkiSpwwxzkiRJHWaYkyRJ6jDDnCRJUocZ5iRJkjrMMCdJktRhhjlJE5HksUk+keSCJLck+UOS3yQ5Lslrkty/r38luc9YSlPtPdOdSa5Lck6SzyXZJclq09Rw2IDl+6fDZukjkKSxWH3SBUha9SR5J3AAzf9Q/gQ4HLidZiDQhcDngNcDC0ZY7bvax9WA9YAn0Axi/BpgUZJXVNUvpln2GODcaeZN1y5JKwTDnKQ5leTtNMHrCuAlVXXGgD4vAN48ynqr6sAB69kY+ATwEuAHSRa093Hs982qOmyU7UnSisLDrJLmTJL5wIHAH4HnDwpyAO3tqJb7ZvFVdQ3NPTNPAeYBb1+e9SWZP3XoNcmjk3wtybVJ7kmycCnLrpnkn9vDvzcluSPJkiTHJHnOgP6PTXJo2+fOdjs/SvL6AX13TPLdJDe2fX+R5OAkDx7Q95T2PayZ5J1JLmmXOayv3x5JTk5yc5LfJ7k4yTv6D39LmjzDnKS59GpgDeDoqprxpthVdec4NlhV9wD/2b7cI0nGsNpHAmcA84EjgEOAW5eyzGHAx2je/xeBjwOnAk+kL7gm2RU4B9gLuBD4MHA0zSHkt/T1fS1wAs29cb8JfAS4EXgr8OMk601Tz9HAPwE/Bj4K/KxnnYcCXwEe1fb7ZLvOg4DvJvGojrQC8R+kpLm0Xft44hxv9zTgLuChNAHssr75u7d7DQc5sqp+3te2HfC+qhpqT1+7h+xlwNnA06vq7r75D+l5viFNkFodeHZV/bCv72Y9z7egCYW3A9v01pnkUzTnHX4A2GdAWVsAf1FV1/etf2+a0P0N4BVV9bueeQfSnOv4BppgKmkFYJiTNJc2aR+vnMuNVtWdSW6gucBiI+4b5nZrp0HOBfrD3DXce8HFUCUAAe4E7hlQ3w09L/cCHgR8vD/ItX17P7tXAmsCHxoQOP93O/9VSd44YE/nf/QHudZ+NMH373uDXOsgYF/gFRjmpBWGYU7SqmLq8Op9hjcBXj3iBRDnjXIYuKpuTfIt4H8B5yY5GvgRcEZV3dHX/Rnt43eGWPVT28eTBmzzpiQ/BbYHHguc19flzP5lkjwAeDJwPfCmaY5I3wk8bojaJM0Rw5ykuXQ1TRDYdC43mmQtYIP25XVjWOVvl2GZv6M5j+3l3LtX7/dJ/gv4t/ZiDWiGVQG4aoh1Tl3gcPU086faB503N+g9rE8TejeiOZwqqQO8AELSXDqtfdxxjre7Hc3/vF5TVUvGsL5Be/dmXqDqd1V1YFU9Gtic5hDoae3jf/V0vbl9HCbw3tI+Pmya+Zv09eutZ9B7mOr306rKTNMQtUmaI4Y5SXPpCzTDkvxNksfP1HFcQ2AkuR/N+WPQXFgwcVV1RVUdATwPWAxs13MRxOnt4y5DrOqn7ePC/hntVaxbA78HLh6yrttprp59QpINltZf0orBMCdpzrR7xQ6kOWn/uCQD7/CQZGeGO2dsRkkeChxJE3Z+Dbx3ede5jHVslOSJA2atA6xLc8HBH9q2w2mGOXl9ku0HrGuznpdfpgnHb0zyqL6uB9FcSPHlEYd5+TDN93PooGFNkqyf5Kn3XUzSpHjOnKQ5VVXvbccpOwA4K8mPgUXcezuv7YGt2rahtcNmQPM/qVO389qOJpicSTPMxqCrN2HmoUmWjOHuEJsCP03yM+B8mrtfPAh4Ac0h0o9X1W0AVXV9kpfTHHo9Ocl32mUeBDyJZvDjLdu+S5K8iWYcuHOSHEVzTuAOwF/RXIX71lEKrapDk/wlzRh0v0zyPZogvEG73e1p9rC+bhk/C0ljlsGnTUjS7EryOJrA8Cyac8jWAm6gGQrkv+jbo5SkAPrP15pq7/EH4DbgcpqBd48Gvt8OHtxfw2E0Q4HM5IdVtbDtP59mWJPDq2rvpb7Je7ezHvDPNHsIHwNsSDMI7yXAZ2jGsqu+ZZ5AE8R2pLkg4SaacHZEVR3S13cn4N+ApwEPoAmLXwfeW1U39/U9Bdhhaee9tbdUex2wDU04vpEm1H2f5rvpHwpF0oQY5iRJkjrMc+YkSZI6zDAnSZLUYYY5SZKkDjPMSZIkdZhhTpIkqcMMc5IkSR1mmJMkSeoww5wkSVKHGeYkSZI6zDAnSZLUYf8P7VCH0b/Sm78AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot score histogram\n",
    "ciderScores = [eva['CIDEr'] for eva in cocoEval.evalImgs]\n",
    "plt.hist(ciderScores)\n",
    "plt.title('Histogram of CIDEr Scores', fontsize=20)\n",
    "plt.xlabel('CIDEr score', fontsize=20)\n",
    "plt.ylabel('result counts', fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'evalImgsFile' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-5658b9b50676>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# save evaluation results to ./results folder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcocoEval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevalImgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalImgsFile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcocoEval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m,\u001b[0m     \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalFile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'evalImgsFile' is not defined"
     ]
    }
   ],
   "source": [
    "# save evaluation results to ./results folder\n",
    "json.dump(cocoEval.evalImgs, open(evalImgsFile, 'w'))\n",
    "json.dump(cocoEval.eval,     open(evalFile, 'w'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}