lambada.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Complete submission\n",
    "1. ablation -> fix the table 3, simply use patch length 2, add define of the patch length\n",
    "2. visual attack, select some of the attacks on LAMBADA and bAbI (explan the table 2 with visual attack results)\n",
    "3. CBT-CN\n",
    "4. compare throughput with GPT2\n",
    "5. compare time/memory/#tokens comparing with GPT2 -> if we have good results.\n",
    "6. why we want to do this\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tai/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from LatentPixel import LatentLlama, TGraph, DEFAULT_BINARY_RENDERING\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "import re\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "from multiprocessing import Pool\n",
    "from functools import partial\n",
    "import torch\n",
    "import os\n",
    "\n",
    "torch.no_grad()\n",
    "\n",
    "import logging\n",
    "loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]\n",
    "for logger in loggers:\n",
    "    logger.setLevel(logging.ERROR)\n",
    "    \n",
    "_ = TGraph.init_render(**DEFAULT_BINARY_RENDERING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQgJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCADAAPADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD0LUj4nPi+bULayZrG01KztY/9JlXfAyqJXEIjIcZuXJbeozBGSD5R3ZOg3fiS10Wyg1N9Z0+0WKFXubTTfPuvMFnaYQqYpCVLtc7nKkhowu5fun1OilZXuLlV72OAvdW8VQ+NYLeMTDTDNbpEWtpWSaFgnmPJstm2ybjIBmaIDapZNuS+3ql/r9pr00djYyXds9vbvAu1dgKySG5yxK4cxmJUDHBZgfurIy9JRQtECVlY821zVPGttp2oPANQW7ENzLpqW2npOLgmafaJePk2QC3ZAShLMQRKwMdaunX+tXHjy4gln1J9OjeQBVtDDAABgbvNtwSM/wAUc7lmwwUISE7SimNHB6nqviqDx4ltGkkemG4gSICCWSKWFgvmM5S2fDhjIBmaMDahZduS5d33jRNET7PDI17YWQgv3eNQLi4LwhpoNqMWCxCZxhCMyIux3Vo17yigDgrC98X6hq8dkJjDZs16kl1JauHiRWtzGR5kEatJ88wTA2Y5PmmMhpdRvvFUk+q6Ppsn+n2cVzdxXG2L98kiOLSPa3C/vC67if8Al0ywxKK7iiklZWFFWSRwHi6+8UXmkala2OnXcSSRXbwtDFl5LcWkqqDgkrKbjy2VRhirpxlZQqa9eeMdL+02trLPex/6NNLepbbGiD+eJEhCRS5VWjgAUxyuFlO5sYdfQKKLAkcdpF14p1HXXiuZFt9NjhjkaUWzqzstxcLsXzI15ZEh3tjAH3FHmK65nhbxH4hj1KT/AISkyW8ToiSRvbzFYrl5Y40jicWyJs3SFf8AWTE/KQxAZj6JRQlZWCKskjk/Emp61Y3GrQWUd2z3OmKmkGG0Mqi9zMG3sFIQc2/MhCdfRsV7R/FElxC9zfXYjvNTvrIxpZov2S2U3BhnyVJL5jiAZvkKuoKFjuPaUUxnnlgNYbxJoMQTUHsIPIMkktoYtreVqCurHYuVUiBRnI5Q5Jfc1/Ub/Wj48t7O3n1I6eHjDx21oUQDGTueS3ZHHcss8Zx8qqXX5+0oqIQUIqK6GdOmqcFFdLfgkv0OAvpfF2kabClg0t40+nmQobYItnskgXCYRm3+VJKxDCQs8WVTGYzt+Cr3Vb7Rpn1dpGlS4ZImlhkjkKbVPz74IcncWwVjAwAOSGJ6SinCPLFR7Dpw5IKC6JL7kl+h8R69r2sw+ItTii1a/SNLuVVRblwFAc4AGeBWf/wkeuf9BnUf/Ap/8a+7KKxhh6UYqPKtPJf5GFPCUYwUXFOyS2XZeQUUUV0HUFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAf/Z",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAIAAAC1TXloAAAGKElEQVR4Ae3Y7XbiNgAA0U1P3/+VUxGBUAxLSb+24U5+JLIsy4xnQth9e39//3H5evvx9v7jfDjH+8xl1Qv+/IWY69ZPPvC1/kkNY/2TK+eyFcCXrvr/LP59vZS3t7cR9/x+mhzP4QR3zX2tfMHBB+yv5To/6n/jlXwp6W8u/LfHFkffc8EYnL8uv/GX4+PTGvOHPdfMvGScPcys+f3CObmvXGf3yfEOtB/ONXNmza877jP7yjXeF4zx6WvjXcvW4LxkHn/+frr08jXH6/s6tWbGwjV5uej8c8yvr3VqzhwOx+SaYQfnd+jxLOZnj09v0penss5eJk5Pf31W2cdrwRgc5tfhGMxla2a/6vbUz5Zd598/3es6f7vvZeZ2zZqZg3U4rrg7vjt52f6nP9dVc3B62uNj3vap73DlWr/Pr8k5WIdjzT7eL3HGnz5yfAl7PLsvrV+L5+/MOFyDOV4L5uCZ/U9rxp/I7YXsRuf+6/th/8PhWHaYOb+AbXrffCw+Lzhc9vDw9i4Pl99/RLeb/IVX8vi+3/fsNej1mJ58Omv9Xfjpfn4/LNj3X5vcrlyn5uXj8HbP85qtucNVh1vPw9t97i67bnXZ/zrzccHh8O4mf39y3eX2Ea3N15o1ww6uQT94BON5jad5XvDxL8VPMx/vtevyeWo+4tP48id1XbIG1z0/7zC2WmvmtuNwTc7xOlzv0Gt+XrIu3Ds4bfv59dy96rr5aXT6C3C77LTVeiaXF7wv2xfs8/vLm+P9FR7OHm5xODsP9xuNmcf3urvDK00++gD3GpwPcvlnAf+zG33pZZ/+UXt5L3rmwu/++/D6QT9jsTUv8wT+5L/tXoYzEOQJFDQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcBY2IVjALWjGNcBY0IlrBLGjFNMJZ0IhoBbOgFdMIZ0EjohXMglZMI5wFjYhWMAtaMY1wFjQiWsEsaMU0wlnQiGgFs6AV0whnQSOiFcyCVkwjnAWNiFYwC1oxjXAWNCJawSxoxTTCWdCIaAWzoBXTCGdBI6IVzIJWTCOcfwDhlmDmOwdxnAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=240x192>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img = TGraph.from_text(['Alice was everywhere, until she'])\n",
    "img.to_PIL(contour=0.3)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pixar113_stage1_path = '/home/tai/src/projects/msc_project/storage/checkpoints/1MModels/dllama_2_b/backbone'\n",
    "pixar113_stage2_path = '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_10/200/backbone'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n",
      "Coder path is none, do not load compressor for this model\n"
     ]
    }
   ],
   "source": [
    "model1 = LatentLlama(backbone_path=pixar113_stage1_path, num_channels=1, patch_len=2, patch_size=8)\n",
    "model2 = LatentLlama(backbone_path=pixar113_stage2_path, num_channels=1, patch_len=2, patch_size=8)\n",
    "model1.eval().cuda()\n",
    "_ = model2.eval().cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAMBADA Evaluation\n",
    "\n",
    "zero-shot performance on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "def save_generated(imgs: list, dir: str) -> None:\n",
    "    for i, (p, t, g) in enumerate(imgs):\n",
    "        fname = f'{i}_{t}'\n",
    "        pname = os.path.join(dir, fname, 'prompt')\n",
    "        gname = os.path.join(dir, fname, 'gen')\n",
    "        p.to_file(pname)\n",
    "        g.to_file(gname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.autograd.grad_mode.no_grad at 0x7f8b4b420160>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from LatentPixel import LatentLlama, TGraph, DEFAULT_BINARY_RENDERING\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "import re\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "from multiprocessing import Pool\n",
    "from functools import partial\n",
    "import torch\n",
    "import os\n",
    "\n",
    "import logging\n",
    "loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]\n",
    "for logger in loggers:\n",
    "    logger.setLevel(logging.ERROR)\n",
    "    \n",
    "torch.no_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pat = re.compile('([a-zA-Z]+)')\n",
    "def add_word(words: str | list[str], corpus: set[str]):\n",
    "    global pat\n",
    "    if isinstance(words, str):\n",
    "        words = re.findall(pat, words)\n",
    "\n",
    "    for word in words:\n",
    "        corpus.add(str(word).lower())\n",
    "    return\n",
    "\n",
    "def _evluate(gen: tuple[TGraph, str, TGraph], vocab: set[str]) -> tuple[bool, bool, bool]:\n",
    "    p, t, g = gen\n",
    "    readable, pad_correct, tes_correct = False, False, False\n",
    "    t = t.replace('g', 'q').lower()\n",
    "    img = g[p._num_text_patches[0]-1:g._num_text_patches[0]-1]\n",
    "    paddle_text = img.ocr(method='paddleocr', square=False, span=True, span_ratio=1.3, scale=3)\n",
    "    tess_text = img.ocr(method='tesseract', square=False, span=True, span_ratio=1.3, scale=3)\n",
    "    \n",
    "    m = pat.match(paddle_text[0])\n",
    "    if m is not None:\n",
    "        text = m.groups()[0]\n",
    "    else:\n",
    "        text = ''\n",
    "    if text.lower() in vocab:\n",
    "        readable = True\n",
    "    text.replace('g', 'q').lower()\n",
    "    if text == t:\n",
    "        pad_correct = True\n",
    "        \n",
    "    m = pat.match(tess_text[0])\n",
    "    if m is not None:\n",
    "        text = m.groups()[0]\n",
    "    else:\n",
    "        text = ''\n",
    "    if text.lower() in vocab:\n",
    "        readable = True\n",
    "    text.replace('g', 'q').lower()\n",
    "    if text == t:\n",
    "        tes_correct = True\n",
    "        \n",
    "    return pad_correct, tes_correct, readable    \n",
    "\n",
    "def evaluate(generated, vocab) -> tuple[float, float, float]:\n",
    "    num = len(generated)\n",
    "    cp, ct, c, cr = 0, 0, 0, 0\n",
    "    \n",
    "    with Pool(14) as p:\n",
    "        results = p.map(partial(_evluate, vocab=vocab), generated)\n",
    "        \n",
    "    for p, t, r in results:\n",
    "        if p:\n",
    "            cp += 1\n",
    "        if t:\n",
    "            ct += 1\n",
    "        if r:\n",
    "            cr += 1\n",
    "        if p or t:\n",
    "            c += 1\n",
    "            \n",
    "    return cp / num, ct / num, c / num, cr / num\n",
    "\n",
    "def evaluate_babi(generated: list, vocab: set) -> dict[str, float]:\n",
    "    pat = re.compile('^([a-zA-Z]*)')\n",
    "    correct_paddle = 0\n",
    "    correct_tess = 0\n",
    "    corrects = 0\n",
    "    n_readable = 0\n",
    "    for idx in tqdm(range(len(generated))):\n",
    "        cor = False\n",
    "        readable = False\n",
    "        pro = generated[idx][0]\n",
    "        t: str = generated[idx][1]\n",
    "        t = t.replace('g', 'q')\n",
    "        g = generated[idx][2]\n",
    "        img: TGraph = g[pro._num_text_patches[0]-1:g._num_text_patches[0]-1]\n",
    "        text = img.ocr(method='paddleocr', square=False, span=True, span_ratio=1.3, scale=3)\n",
    "        raw = text[0].lower().split()\n",
    "        if len(raw) == 0:\n",
    "            raw = ''\n",
    "        else:\n",
    "            raw = raw[0]  \n",
    "        m = pat.match(text[0])\n",
    "        if m is not None:\n",
    "            text = m.groups()[0]\n",
    "        else:\n",
    "            text = ''\n",
    "        if raw in vocab:\n",
    "            readable = True\n",
    "        text = text.replace('g', 'q')\n",
    "        if text == t:\n",
    "            correct_paddle += 1\n",
    "            cor = True\n",
    "        text = img.ocr(method='tesseract', square=False, span=True, span_ratio=1.3, scale=3)\n",
    "        raw = text[0].lower().split()\n",
    "        if len(raw) == 0:\n",
    "            raw = ''\n",
    "        else:\n",
    "            raw = raw[0]  \n",
    "        m = pat.match(text[0])\n",
    "        if m is not None:\n",
    "            text = m.groups()[0]\n",
    "        else:\n",
    "            text = ''\n",
    "        if raw in vocab:\n",
    "            readable = True\n",
    "        text = text.replace('g', 'q')\n",
    "        if text == t:\n",
    "            correct_tess += 1\n",
    "            cor = True\n",
    "        if cor:\n",
    "            corrects += 1\n",
    "        if readable:\n",
    "            n_readable += 1\n",
    "    return {\n",
    "        'paddleocr': correct_paddle / len(generated),\n",
    "        'tesseract': correct_tess / len(generated),\n",
    "        'union': corrects / len(generated),\n",
    "        'readability': n_readable / len(generated) \n",
    "    }\n",
    "    \n",
    "def process_lambda(raw: str) -> str:\n",
    "    raw = raw.replace('`` ', '\"').replace(\" ''\", '\"').replace('``', '\"').replace(\"''\", '\"')\n",
    "    result = ''\n",
    "    for idx in range(len(raw) - 1):\n",
    "        c = raw[idx]\n",
    "        cn = raw[idx + 1]\n",
    "        if cn in ',.?><{}()\\'' and c == ' ':\n",
    "            continue\n",
    "        if (raw[idx+1:idx+4] == \"n't\" or raw[idx+1:idx+4] == \"'re\" or raw[idx+1:idx+3] == \"'d\" or raw[idx+1:idx+3] == \"'s\" or raw[idx+1:idx+3] == \"'m\") and raw[idx] == ' ':\n",
    "            continue\n",
    "        if raw[idx+1] == \"'\" and raw[idx+2] in 'abcdefghijklmnopqrstuvwxyz' and raw[idx] == ' ':\n",
    "            continue\n",
    "        result += raw[idx]\n",
    "    result += raw[-1]\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset and Build the Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88c174f29d734a9ab8fbbc5d984bd7ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/7.32k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59b249702c6d41d0aa35346265a419cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54ddd46909144161813156afc943a020",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/269M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23ad83c645214bd8a55eca23c1e89d9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/281M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27417f5dcb5d43329a2869d2e29cbeb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/1.14M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d27a8cd34e7440d6b3dce56274ce90e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/1.08M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e3f46c31c3455a958f7ae1ad090c05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48813a97f830458b8053c3c4c943643f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/2662 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b99775b04d25463b935e08468baf989b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/5153 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43063311b82549cdafd99cce86221811",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/4869 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "333332\n"
     ]
    }
   ],
   "source": [
    "data = load_dataset('lambada', split='test')\n",
    "TGraph.init_render(**DEFAULT_BINARY_RENDERING)\n",
    "corpo = pd.read_csv('storage/unigram_freq.csv')\n",
    "vocab = set()\n",
    "add_word(corpo.word.tolist(), vocab)\n",
    "for a, b in data:\n",
    "    add_word(a, vocab)\n",
    "    add_word(b, vocab)\n",
    "print(len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 1106\n",
    "txt = data[idx]['text']\n",
    "words = txt.split()\n",
    "prompt = process_lambda(' '.join(words[:-1]))\n",
    "prompt = TGraph.from_text([prompt])\n",
    "prompt = prompt._spacing_text(3)\n",
    "prompt.set_device('cuda')\n",
    "target = words[-1]\n",
    "gen1 = model1.autoregressive_generate(prompt, None, 4, 'gray_scale', temperature=0.1)\n",
    "gen2 = model2.autoregressive_generate(prompt, None, 4, 'gray_scale', temperature=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tabitha'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf6K838KWkul6tba5rQnt2fw/pNiqPZJGkU0sjo0KhYwUw/l5UHA8z5hgJt0PHGo+JbL7b/AGL9uWRNPMlglpYrcrcT/vPMEuR8uxRGyfMu5mIAlI8upNOv9auPHlxBLPqT6dG8gCraGGAADA3ebbgkZ/ijncs2GChCQlfVk1ex8Z63eWUupMJ9MsQuy0Ekcca3EgnMbCM7pY42LqhLFjIflcBVGGt34lm8RxXd6+uRWMH2y3S/i0dTOtsyWLRt5YVvmdw2T5bH748uPafKuRw6/c6ro19qsF3Yzx67C9zFY2asmX0xFZmYI5ZBKXi8zcQAxBb5VKmjjxXonhPQrVWu5RLo8JuFFkA2niNrZCI1wWMohkmba+8s8XyoBlD0Hh7UNak0jXptQe7zb3DCwmutOKu8AgjZHMMeGclixKjaxOV2xn5Fw9O1PxK+kvLdXGuLCbtI7i5FmsksEXlucwxm0idmMnlK2YXUK2VOQ5SvYah4uZ5rnVIL6z8zZHd3dtpaG4ijW4v9vloFk34xbDb+9ASQsD8xkrpPAVzMNHl0/UI5INWS4ubu5t3hMZVZru4KNjcww2xiAHbgDkggnn7W+8R6hJOZ/wC0ruNb2zKrLp/lwBTeRbmEcttHIpCBmGHm2AEs4YIzWNIuvGEdrp093c6lcSfZ9Mnninso1Dvcy+XcxsFjUgQqu8AEFSxLllwBJNHro8I+BJrPTd2p23lvNAYBGsLfYJ1bKDaq4ZgAhKKWKruTORYhufEvn3F7HeX09nBqFlBbW82nrG9xBKlsJpZPkVvl8yVvlCbGV92VGxcfStb8cO8xvo50kj8ma6hFpK/lBbiLzkhP2ZEZfKMw2iSZ2wuxiQWbrNLv9fu9ehjvrGS0tkt7h512rsJaSM22GBbLiMSq4U4DKT91o2bLvNV8XRaZqcEFhO90n26GGcQoSJnkY2RUZw0YTYHYggF03H5ZilO2TxXo9hLNpqSXMtxqeqpDp8tsFiRS91NFI7HDZaRYwGLKhSQDbnDnoPC91fXFxerJc6leaeiRGC51Oy+yzmUl/MTZ5ceUCiIhtnJdhuOMLyenHxjoXgvRre0We5mk0SNpIntfLFiYzbptXCM3meVJMxDiQs8XypjMZ3NHvfE194F1lw0jawiTpp7SwtHIX8oFN/mwQqTvJwRGFwADkhiTUdZ8VzkNpGlSAs81xarPEEWWAW0gRZtzAo5uPKO35G2OvQrKEx5tZ8TRpqawXesyWapaixubjR2jnlLNP5v3IHKkFUG5oANoUbQXWVuw8M6i1zYQ2l5NdtqcdulxOt2iiQJI8gRjtRBg+W2AVVgAN6q2RWHZ6r4ul0zTIJ7CdLp/sMM05hQEzJIpvSwzhYym8IwABKPtPzQl8/QYPF2l+F9DtYHnmm/sRJzbSWqRLDJAbfbahmHyNKjSIzSFiGBZQoUrXQQ32s6r4N1fVNMkzdXsU02jJtT5E8vEBy2Ad5US/OBt83afu1T1HWfFc5DaRpUgLPNcWqzxBFlgFtIEWbcwKObjyjt+Rtjr0KyhK9rq+tW91FcJNrN9ocNxGbi4u9LKXTBopw6CFYUdkV/spDLHnLv8xVWC56T+I11e81GdNStre6SFZLy3sd88VqJ9QaLZFsbL7WtVZSjMqyEsFOWGpo8msWHw2uppo57PVTd3khUWTu4Z7uQ/LGnmkZDZDASquQ2JFGDn6Va+ItX1bR7ia+1W0YWl+kt5NbRl1UyWpVY90EQXJQ/62HcdsmARtcR2mv8Aja41vSXvLSSziuEs2e3FrMYisiRmbeFt3KOrtKo3Tx7diFlxkvc+I8muzadq9hYx3zwzaU6W0NpZCdbqRhIsyyn+DanllPmQlmOBKRsrQsL/AF1/FCwzPfHN3OlzavZhbSC1Ak8mWObYN8jYgyvmPjzJPlXb8keo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz011nxTNplgkcV8t1a6esesu1gVK3Bkt1d4cpiVlQXTL5QdCQvDZQEt7TUdV8Q6XLZa1rnkpaX0c1/e6WsEyZe0KxoHhRBkqSGZGyBIByMpqXWp61N4Q0O9lju7Ge6SF9UaytDJPahoSx8uJlck+bsQgq5Csx4xuGfeJr+tfB/XI9RS7j1iW3v08qO2VJGAklEcYjG8YZAq8FiQ2QxJDVHYRa7aX2p6na3F9Lby6rYhIJrEI91HJDaRSzSZUMNq7jhBGFZH3ZHyrX8NzeLotFtbm4M8S20um20emrpyRKY5IbVZ2YBQQqGSUgJtCMjBtyjasmkXer6xr1ro2vm7ktLrTJp761uLUQot0klvvhRhgywLvK8hlcOQXlBIToND8PWh8K3NjcW0cdnqiM8lhCjww28ciBTFGpCsgxyeFJdnbahbaMPXEv8AQr/VV0WKe1hlitpElgsZJnklknu5JgJFhmK8srfMjqu7ACGQMNj7ZqkPw/8AtTS3y6kYs757bzZsluyRRZGQflJhyowzxkhkrn9KtfEWr6to9xNfaraMLS/SW8mtoy6qZLUqse6CILkof9bDuO2TAI2uI7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJe5490e9vNYXUbWKe5a18P6oIIPskc8fnssaquGjYlnDN8ufmEeAMFw2P4y1HxLNb32mf6c327+0bWSzFipi2C1uWtlhkxudpFRHYKZMMMExfcbU8T22vtZ3dnPf6lPFb3Gl3wubOwVm3G6PnJGgjfKRrGkgXDuONzODgxpN4uij1+a2M9tDYRXN3aWkenIPtk/2u8Kpnb8yuiwlgo3tvRg6liXuLrOvx+M9MtLiW7jgudTuILiB7NRAkSxTtb+VKQCxdUV2wXwwwTF9xtSXVddstav0ewnuLCK7WbekIYrZmBFxHtILyef5jFcM2xG4y0Qbl7S08RSXug3Wsi+guoNbgN2bWyjYSM2loryOUjOV80vGXB2gORkbVKbHgWDXdM07w3Y37zyW8uiBpY5LURrZSRCBUjHG4MVd9wckloyVCAFar3OreKYtejVWvkh+13Qu0/sozW8EKxTm2ZGXDybwsbMqlzvwu6IkI2fZL4l1mTSGmudVSSPVc/ari1UrBH9kuVZlDW1uwb5gNzoyBmjwXO9K2J9FfX9MtRqlrnUzdz2Md/PAxmtoo5JttxEAu2KZ0RGEg2LuZG52pGeosNE07S7q4ubO38qa5z5rb2bdmWSXoScfPNKeP72OgAHn+nHxjoXgvRre0We5mk0SNpIntfLFiYzbptXCM3meVJMxDiQs8XypjMZ6DQNS8SXHg3VJ3i+06tD5osUuYZImlYRhkEnmQ2+cuSMqirtwMkhjWHNrPiaNNTWC71mSzVLUWNzcaO0c8pZp/N+5A5Ugqg3NABtCjaC6ytY0htfXxHDdTxaktvc2Vil1dzQK824T3RCbVjQBDvQM2wOiMu5ELM8dPS9b8cSLeLfRzxzeUjXCC0lc2x82MSm3JtkRtkbTELvuCxRNu8bt+54Ju5k1HWbfUTdi7vr17m1a6tTA9xbx29rH5pUcA/MgP3Tkn5EIKLn2yeK9HsJZtNSS5luNT1VIdPltgsSKXupopHY4bLSLGAxZUKSAbc4c17zXdftpbKKyv8AWZtPuL2OFru70pUuyDBcvMqQmOPIRY4XUhGJYsAJceXUlvN4uutctEUz2kdz5MMt82nJ5jW4/tAqzkqNshVbZsEAI8gygDFDX1/X/F2naDqVz5l9bzWVpdfZJE09JFuWilnRpJyQFTEKQyqQUVmdtqyDEddR4o164j8Jy6hoEu+6XUIbRD5Ibc/2tIJE2uVB53r95fUMOGrLsLrxXe6pb2dvc3Y0uVLkDUb2yEU+wfZtrlDGoEu5rhEVlQFQHKybfnjubzxdc69HY291fWnn3d1DO/8AZqPb2cYin+yyJIQN+7ETsMuN+FJizsanq+seKNT0r+2NPsZIpIXmigtY4EumguIrK68xw20nP2grB0wfK4JWXB1NQufEun3ktnLeX01ifs81xqNpp6vLCHFwHSCMI+5Q8duNpWR1WVizHhlLC/1nVfAttc3kXn6kNbSKZPsqHYkepBG+QFwNiKeQzbdu7cSN1Gg22s2XgTxDbWU19LrKXepG0a8gSImVpZGiZcoqMrFlfdyuXYZAG1ZNR1nxXOQ2kaVICzzXFqs8QRZYBbSBFm3MCjm48o7fkbY69CsoTHm1nxNGmprBd6zJZqlqLG5uNHaOeUs0/m/cgcqQVQbmgA2hRtBdZW6QXWsp4EhuE+3NqDbDM80SNOkRlHmOqLGoLLGWZFMYY4UNHuylZces6/b+B/GOoXUt2s9ik76ZPdWawu0S2qOkhTGCS24nIHOQVTHlrH/afiXypPMl1VLHzYhd3a6YoubZysplSCII++NXW2UMFm4lkPmPt3JXvtW8UldLiWbVVaTfkWtiUmkj81lid2e2kiEjRhGKMYAjE7sBgI49cvvEd1deJLaD+0rqBLK7EFuun4gZhEwRWSa2/eDdgfJNJvbBCBCwSnK/iW30fXrK2tL6Rnl1dlsptNVoGgZrl45VYhvMkaUxqEJ+ZG/1RGJDHdabq/2PVx5OpWkUep29yslrYB5ZF/te6diMxszhUMcoUA44OMOwbsItU1ePxBPp1y8hg055r64njiDtPauG8iIRqC2dxkXK5Y/ZOR+9Fbl/omnapdW9zeW/mzW2PKbey7cSxy9ARn54Yjz/AHcdCQeLtr3xpb6Jpc1zNd3NxfaYk98RYKrWLB7dX8tAufN8qSd9jb9zxfKgGYz0HhrVppXexvri7leR5JLCW9tTBPcW6LEHeRNiBSsspQAqpKhSAeWPN61N4u/4R7UZyZ71rqXVLRLA6cjIkCpcmByu0l2YxxKCfkZZFGwsdxPEMuqX+k+JLK5udc+3S2l/FDpkGmb7SRPLkEOJhCcsybGx5v3ztwPuDsNEvbyXzYtTSdL5pZC0XkEQxKuwBY5AMOpDKwYncxL/ACptaOPDutS8UC/1DS7BY57ywS4vY3kCBbqORG+ywlcjaPMLruDZ/wBEy2BKKr2ur61b3UVwk2s32hw3EZuLi70spdMGinDoIVhR2RX+ykMsecu/zFVYLIB4g/4lupWcG3UtR1CW3uJbmFgq2C/a3tjIgGV2h4zj5GZiEZlJJFe71HxZZeNLeyDzvp6y28Ucj27ulxEQgkkk8q1ZVk3GQD97Eo2qSm3l9DxFP4lj1HWZtMnnS1sNKju7W3jtFk+13QM5MW4gkqQkYZV+blNrJzujgutfufEDWrXOpQvJcXEVxELJRb2tsBJ5M0MzR4eU4gJUvIMyPlBtITPh06+n+AB05DqQvxoTQ+U9tsn3rGQYfLZAcZHlj5clcEEkhqksItdtL7U9Ttbi+lt5dVsQkE1iEe6jkhtIpZpMqGG1dxwgjCsj7sj5Vx9Tm8XaxoerafOdVKz+H7x7mMacihL4eVm2ibbmSPBdVIB3BmKySEZj6TxZFq9zouhTaH9rudQieaSCaaAI4kNhcrG8isqqhLsgIZVALAEDpUeoa3rF5eS3Fh/atroR+zxSzppji5hbFwZGjhkiLvlvsqk+Wy4ZsYIZlr2D6ovitbqCTVZ9NuLSyin1C6sPLuGKz3WEMRjQKuWVWYJlUZTgBjKmpoWqavLevBqTyFNKQWN26RBzd3TuuyXagzGPK8uQgZUC5IJHlGjXfDlppnw78RadoljIDPpkqJDGXkZ2W2EKAAkknbGi4HXHqTnL1688Y6X9ptbWWe9j/wBGmlvUttjRB/PEiQhIpcqrRwAKY5XCync2MOupBqGvSeAYrthIdSLhZXjtnEqQ+dteQRyRozSrDlseWAzj5UIIU17PWbmxuLS5ku/EGoaSUuI53utHdZxPmExARRwI+zaJvm27ckgtnaBHoE/jFrK9k1yf7M1lFGf+PTznmlNnEXGyMDfGspkOIyWdvlBUJh8dNW8Yjw9NcRHVZZDdxxyyzQ4aKLY5JiX7Cshbf5atmCRdrcEEOUr69e+LNS8F3lvLHfPNcaVNHBFa6a7i7cmZJFm8yONkZYhEykCLc7NtWQYjrtPFF3q9tcWS6ebtIGSUl7S1Fwz3AKeTE6ngRMDJuYlANq/vI888nHr/AIuhs9Glv5L6C6ll0uKaJtPQQtDKYFmkdyBsmMryR7ASQuCIh/rADUPF11e6dDcQX08LfZ5b6OfS08u3uheWnmRxtt+aNEMpV8HgMwlkK5j3PDV74jt00STXZru6/tDTBLdj7BsFrcBoFRAEXKlvNcvvJGYywEaggXPGWh29zour6pBYfadXj0+UWw2GXc6wzqi+Wcq//HxKNpU538g4GDXNO1G+8ZaO1ldz2MK6fepNdwwK7LmS2KqpcFFYlT95WyquAM/MvPrrnjG4/syeS3vra6nl04zWcOnboFtpPJM7s7DKSB2lQruJVFz5Y/1ouXd940TRE+zwyNe2FkIL93jUC4uC8IaaDajFgsQmcYQjMiLsd1aNdTwjc6xqnh67XU7ieK4814oZzA6TKpRSGIlt4lZgxbBEW3AUHcQxPJ6Rc+I9O8HaJa27azFPBo8Is4DpuftF4N6tb3OYsxRKVhUMTHlWY72xuXrPFdtcS614TuYpr5IbfVSZltYBKAGglUM/yMVXJCFuABISSCFZePtb7xhqpW11i3nLC7sZ54Ps0xEMiXtuWET/AGaNPLVfMJ/ezHCqQxAZm2LbS9Ufwj8O7WAT2V1beQLiQ2+57YCwmRiVPCtk7QWBAYrkN9014H8VXniGG3aS+so3lW1m1CGwiEjxRPqIVmZ42X5vLtmPG0GX5Qok50NIvbzUbUaT4rSdodc0+AiKaAxYnninae1UqBtVEiyNxLjdyxJWtDxPpLxeGBaaNDsmbVbW5XMbTBXa+jlkkZcglQSzkAjAzyAOOf1vUfGOns9nbvfS2lvdvF/aIt/38w8qB0yI7WZSu6SdSVhUfulBYNkPua3qWsx+CrG6ikkh1OZITN9ksp2O4rlgq+TK8Qz3kibgbSFZgy8/q2q+MLfw+ZLM6z9tGmNLZj7BHO1zOTJ5glxGgQIgiaMERMd2GSRwYqp+IX8Sy+GtdsI7S+RZ4tRW0t7TTVdbsvcXIcS4A2Yi8p0bchdnJ/fH5D2Hiu2uJda8J3MU18kNvqpMy2sAlADQSqGf5GKrkhC3AAkJJBCstfwtf67c6jGuoPfPutGfUI7qzEMdndZTEVu2xfNj5m+bdLxGh3c5bPkl8SR6vdjR/tS3Eupyb0vtORLd4hby7HMkYBcGRIk3bgwVYwVP3pH6bq/jHUHhEdrBDbib7Mst2LhZWdUfdJIhtUDIRtcBTGN67fMG7aOwnluPsdu7QyJK7IsiwtuaItwSOMMATzkYwCe2K5u5HiqeTUV894Y4NPWaFEiUC4n8yb5NwBKrsWMMFy2HXawIJaK91LxhPp9/FYaW0eoLcXiQsyqqCERzC3kUtIQzNIIgQ3QE5RRtY9Tp1vcxwma+ZPtkrF5VildokJwAq7j0AVRkBQx3NtUsRSXUmoQrM8It5d0sSwoUcFVLKHZiM5xkkAADjkgEkV5ZrqyWK1la6lj+xyPLepGJJFdNgGFSPBZtzHAX+E4XHTWooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooor//Z",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAJrklEQVR4Ae1Yva5e1RE9H7JSOU8Ql0meICIVRVpEFadJ4A1AckFBehIheiQKkJIiqVKkSJc0uAgFJkqHJSQaaFIgIaQUMZZ1mLVmZv+cn2tfx4MGaT7bZ++ZWbNm9qzznXOvL+syfy7rclm2ToVIaNwAyb8zQbcU37IQ6AZ33exZ290GszG36J29xT+3ReBQfrBtbLYfizoAdFffzazR1u7AFxH4YlUv3F7kA4cZjMLjSEYVChTRNIFxKgtIDkLM1iJkAUrz6GSSZmrEwxptaLg9gRVBop4eaNUYuNxgwnRZ/S6QVUBuLfCry29lIuk1v6Pd9BWduaTzFwFuoJxqAPp2Dm/QzgyQJ/hpwG3s7lqWgwP3oNzc8n0WIm8WoZ1tePiHqJrqmAIG14U9KTvTp6AZY3FxnaKl4rY3MBAv89TjP+7A6EIyDIx83iujTaf5ZXG09+9rCzieq7ehiXC1zqYtweNFjz0lbHsdmIbME4V18hyZ8Si3T9Ef5H5y8+s4Vr8xbCVooBnKC8JahzjLCtNcwxYRCfdD0qTtKG1Ai5DSA1bMSJ18bIF7b2MXeBrHMyV7ggYO67lz95QWQh0mpmWfYXuFy0Mqc7eohFAoy4bLn6r+yB/SPKO79ElM22gabQe1O0FjXs9vj5Nbesx/tnsvrKyYu89+vGVPasr9vovsPQY5DqjCGIa/HDkSf5v5i422q8E3m2WAXPwQSDa2x6JVLSARxgHAHzP8SSBYHAVRr2EbQwJvqeS2fCWbCAnrYAfSwyZEYU54GjVcVkPHBJtfe8dKtGXQB0b52F4W7rDS78nNC8dUYkyfeZDDPH3wWCIWPuO0KttTwt7GrqoQnd7SfGr6w9GOzaZ40ajamwconMPdNCDd21Y0yKMokeSJh1HbKLKdYwczx0ToIF89KLawrjek4lTTcVZF4C08pE49jBhPb+uEbF5uGBpIxdmo2kadB4mzi5bXOjkSb6kbchycWEfqE8YomI2Lt0QPDI3CZEgxajaMtqNIbcZpNGIkwzjhb+XaRpznnMhotK2WHkf9PWwFL4MHFb/bz1MXP0s88w/Hak9p+PiI1AeqPVEGZN9i/MPHzI1XAHvPkIVtU2eHFMfONyS3xME3EUq25mOdyMaHFubzBDPaFDk3z9raZ+yQO8c+52qPE2D1PTOeG154Gmgv9+kdiHcnJsWbgOPXHUFWuycIlmPV1XKB8oBywVSnBRoSAYT0gutcfwSSwmsTqVhgmK9khMmvh3pKEZbaDgLTBdsDSLdhMQYTG/SFD51AaBS75ptBDcI8vXi2I/u7oxExraP7jlh/NBOOGBkV1MkWKCzOfhBFNJcpCq9gVpOVZ2FiSx02Lbn7jN8d1hQZ0dhBBnzzh2grKhGjtAVQ7c56bAAnkac4KkFhK99JHDN7bDCC5sC8kksyJI2U6m6g6XTNO+TufK4cb6WhWwP25nlEdGflx0gvcEPyvPV9n3h5aVzfct2Sn+oQMXdngFss+FuREdQD6tXhOdR8arJhzFb/gdPrkx+GVZKQ1NPohFEmuRqNfK97Xy147c2OZOeQgvTtA8P9pXUPIFc0dB00sXwP2xPxiFfHKBM6CppvX3SYYkuECl12dU8uq2G51lNLP6k/VCLiBE83sUf/ATBybwnH2Pdo34/BA+OLLhPFGPpTWR3iExcHxItqrV7f89vjLnAMVE0oZFsFAOQPP3Qj4jXMIR1Zk3NbBCpGrvjLTE1XDi3EkAJYzZoa/tdSbkz9ovkdSocweuE+JVCK5RF0PoFpKEK/uvpocrf0Ym5k4gPsuKrlrqEtByqAhJ5HlFDLB/75ak4p02/p6TxImx2tFB6Wc0jZpIJPeBcGXSc8Ceu5t/QKbtTKpFe04T8ZWOlGrZv9dVB4pDrYbzTZIMjcqm2CW1Nl2HrFPq9xSO1F5eAeb9QmNYs0J6zxlweWhBCTgv50lW7UzwVevCFxtY8DBxMxI3NOQRFotjJ4KqhJ7+iBi4UUqEmM4TDah/9QAAdrjFfjkWY2zBo4dHpOO0J3PNvdY6pfq9ieC9/h9tB07ZtoA7nHxMVHgobsaavBAUJOUg8U83bMGTNn1P9p7U/iD6195Pqlrsdxij4IfGDNPFjeWu4vy4MPYd9c/m7uay2rfuQbsPK7jwXfCjjkn3AhouveUJwlioFUTWCWBRoLNiRFwArtyN9d/oHY+ldc7PPZH9ffrz/+6nU6/inOWx7x9YeffyrbL151+2x1Sq5sZuhEj6MQuWKzMRTBQ2KrIMWwYmPUxJ7t6O5x8v+B5I2fmrk8Is/D9dUP119L5PZ66xV0wVoMra/9dl3/ffehGJ+o48qrFrSDwFCmdlVJ6GfPipcrAq2uJ8JvXFrVAp1FJ6BlNAigJcGxfkz7Ry+zwLLege/NdX10h/VvWhfgsc9dWZfP763rlz9x1xWr53HFhUdtwnh47skbNDSDTBS4/vWKjdEdluKzsr4tCej1BRr31rc15S9cfrf+7c+gXv+Au+l91UMBvN7n9eXBc7b1h5bOk1f54QX62eu2/YyDd1v7bVKN04frJoBEurBK30bjqxQbyH/x3uVfLy4/e/6Ny1f4AfFX7Onr5aXfLBD9P8s76y9vY7r9899l+ROty/LRz7v7eLd/UR3jrueNYdUevvnB9XrZoCNb25TKYR7c0jkai+qiDhw12Sy8pXAWJaL6KIWjJpuFtxTOokRUH6Vw1GSz8JbCWZSI6qMUjppsFt5SOIsSUX2UwlGTzcJbCmdRIqqPUjhqsll4S+EsSkT1UQpHTTYLbymcRYmoPkrhqMlm4S2FsygR1UcpHDXZLLylcBYlovoohaMmm4W3FM6iRFQfpXDUZLPwlsJZlIjqoxSOmmwW3lI4ixJRfZTCUZPNwlsKZ1Eiqo9SOGqyWXhL4SxKRPVRCkdNNgtvKZxFiag+SuGoyWbhLYWzKBHVRykcNdksvKVwFiWi+iiFoyabhbcUzqJEVB+lcNRks/CWwlmUiOqjFI6abBbeUjiLElF9lMJRk83CWwpnUSKqj1I4arJZeEvhLEpE9VEKR002C28pnEWJqD5K4ajJZuEthbMoEdVHKRw12Sy8pXAWJaL6KIWjJpuFtxTOokRUH6Vw1GSz8JbCWZSI6qMUjppsFt5SOIsSUX2UwlGTzcJbCmdRIqqPUjhqsll4S+EsSkT1UQpHTTYLbymcRYmoPkrhqMlm4S2FsygR1UcpHDXZLLylcBYlovoohaMmm4W3FM6iRFQfpXDUZLPwlsJZlIjqoxSOmmwW3lI4ixJRfZTCUZPNwlsKZ1Eiqo9SOGqyWXi/BSxicCf7e5qvAAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen2.to_PIL(binary_method='gray_scale')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf6K838KWkul6tba5rQnt2fw/pNiqPZJGkU0sjo0KhYwUw/l5UHA8z5hgJt0PHGo+JbL7b/AGL9uWRNPMlglpYrcrcT/vPMEuR8uxRGyfMu5mIAlI8upNOv9auPHlxBLPqT6dG8gCraGGAADA3ebbgkZ/ijncs2GChCQlfVk1ex8Z63eWUupMJ9MsQuy0Ekcca3EgnMbCM7pY42LqhLFjIflcBVGGt34lm8RxXd6+uRWMH2y3S/i0dTOtsyWLRt5YVvmdw2T5bH748uPafKuRw6/c6ro19qsF3Yzx67C9zFY2asmX0xFZmYI5ZBKXi8zcQAxBb5VKmjjxXonhPQrVWu5RLo8JuFFkA2niNrZCI1wWMohkmba+8s8XyoBlD0Hh7UNak0jXptQe7zb3DCwmutOKu8AgjZHMMeGclixKjaxOV2xn5Fw9O1PxK+kvLdXGuLCbtI7i5FmsksEXlucwxm0idmMnlK2YXUK2VOQ5SvYah4uZ5rnVIL6z8zZHd3dtpaG4ijW4v9vloFk34xbDb+9ASQsD8xkrpPAVzMNHl0/UI5INWS4ubu5t3hMZVZru4KNjcww2xiAHbgDkggnn7W+8R6hJOZ/wC0ruNb2zKrLp/lwBTeRbmEcttHIpCBmGHm2AEs4YIzWNIuvGEdrp093c6lcSfZ9Mnninso1Dvcy+XcxsFjUgQqu8AEFSxLllwBJNHro8I+BJrPTd2p23lvNAYBGsLfYJ1bKDaq4ZgAhKKWKruTORYhufEvn3F7HeX09nBqFlBbW82nrG9xBKlsJpZPkVvl8yVvlCbGV92VGxcfStb8cO8xvo50kj8ma6hFpK/lBbiLzkhP2ZEZfKMw2iSZ2wuxiQWbrNLv9fu9ehjvrGS0tkt7h512rsJaSM22GBbLiMSq4U4DKT91o2bLvNV8XRaZqcEFhO90n26GGcQoSJnkY2RUZw0YTYHYggF03H5ZilO2TxXo9hLNpqSXMtxqeqpDp8tsFiRS91NFI7HDZaRYwGLKhSQDbnDnoPC91fXFxerJc6leaeiRGC51Oy+yzmUl/MTZ5ceUCiIhtnJdhuOMLyenHxjoXgvRre0We5mk0SNpIntfLFiYzbptXCM3meVJMxDiQs8XypjMZ3NHvfE194F1lw0jawiTpp7SwtHIX8oFN/mwQqTvJwRGFwADkhiTUdZ8VzkNpGlSAs81xarPEEWWAW0gRZtzAo5uPKO35G2OvQrKEx5tZ8TRpqawXesyWapaixubjR2jnlLNP5v3IHKkFUG5oANoUbQXWVuw8M6i1zYQ2l5NdtqcdulxOt2iiQJI8gRjtRBg+W2AVVgAN6q2RWHZ6r4ul0zTIJ7CdLp/sMM05hQEzJIpvSwzhYym8IwABKPtPzQl8/QYPF2l+F9DtYHnmm/sRJzbSWqRLDJAbfbahmHyNKjSIzSFiGBZQoUrXQQ32s6r4N1fVNMkzdXsU02jJtT5E8vEBy2Ad5US/OBt83afu1T1HWfFc5DaRpUgLPNcWqzxBFlgFtIEWbcwKObjyjt+Rtjr0KyhK9rq+tW91FcJNrN9ocNxGbi4u9LKXTBopw6CFYUdkV/spDLHnLv8xVWC56T+I11e81GdNStre6SFZLy3sd88VqJ9QaLZFsbL7WtVZSjMqyEsFOWGpo8msWHw2uppo57PVTd3khUWTu4Z7uQ/LGnmkZDZDASquQ2JFGDn6Va+ItX1bR7ia+1W0YWl+kt5NbRl1UyWpVY90EQXJQ/62HcdsmARtcR2mv8Aja41vSXvLSSziuEs2e3FrMYisiRmbeFt3KOrtKo3Tx7diFlxkvc+I8muzadq9hYx3zwzaU6W0NpZCdbqRhIsyyn+DanllPmQlmOBKRsrQsL/AF1/FCwzPfHN3OlzavZhbSC1Ak8mWObYN8jYgyvmPjzJPlXb8keo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz011nxTNplgkcV8t1a6esesu1gVK3Bkt1d4cpiVlQXTL5QdCQvDZQEt7TUdV8Q6XLZa1rnkpaX0c1/e6WsEyZe0KxoHhRBkqSGZGyBIByMpqXWp61N4Q0O9lju7Ge6SF9UaytDJPahoSx8uJlck+bsQgq5Csx4xuGfeJr+tfB/XI9RS7j1iW3v08qO2VJGAklEcYjG8YZAq8FiQ2QxJDVHYRa7aX2p6na3F9Lby6rYhIJrEI91HJDaRSzSZUMNq7jhBGFZH3ZHyrX8NzeLotFtbm4M8S20um20emrpyRKY5IbVZ2YBQQqGSUgJtCMjBtyjasmkXer6xr1ro2vm7ktLrTJp761uLUQot0klvvhRhgywLvK8hlcOQXlBIToND8PWh8K3NjcW0cdnqiM8lhCjww28ciBTFGpCsgxyeFJdnbahbaMPXEv8AQr/VV0WKe1hlitpElgsZJnklknu5JgJFhmK8srfMjqu7ACGQMNj7ZqkPw/8AtTS3y6kYs757bzZsluyRRZGQflJhyowzxkhkrn9KtfEWr6to9xNfaraMLS/SW8mtoy6qZLUqse6CILkof9bDuO2TAI2uI7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJe5490e9vNYXUbWKe5a18P6oIIPskc8fnssaquGjYlnDN8ufmEeAMFw2P4y1HxLNb32mf6c327+0bWSzFipi2C1uWtlhkxudpFRHYKZMMMExfcbU8T22vtZ3dnPf6lPFb3Gl3wubOwVm3G6PnJGgjfKRrGkgXDuONzODgxpN4uij1+a2M9tDYRXN3aWkenIPtk/2u8Kpnb8yuiwlgo3tvRg6liXuLrOvx+M9MtLiW7jgudTuILiB7NRAkSxTtb+VKQCxdUV2wXwwwTF9xtSXVddstav0ewnuLCK7WbekIYrZmBFxHtILyef5jFcM2xG4y0Qbl7S08RSXug3Wsi+guoNbgN2bWyjYSM2loryOUjOV80vGXB2gORkbVKbHgWDXdM07w3Y37zyW8uiBpY5LURrZSRCBUjHG4MVd9wckloyVCAFar3OreKYtejVWvkh+13Qu0/sozW8EKxTm2ZGXDybwsbMqlzvwu6IkI2fZL4l1mTSGmudVSSPVc/ari1UrBH9kuVZlDW1uwb5gNzoyBmjwXO9K2J9FfX9MtRqlrnUzdz2Md/PAxmtoo5JttxEAu2KZ0RGEg2LuZG52pGeosNE07S7q4ubO38qa5z5rb2bdmWSXoScfPNKeP72OgAHn+nHxjoXgvRre0We5mk0SNpIntfLFiYzbptXCM3meVJMxDiQs8XypjMZ6DQNS8SXHg3VJ3i+06tD5osUuYZImlYRhkEnmQ2+cuSMqirtwMkhjWHNrPiaNNTWC71mSzVLUWNzcaO0c8pZp/N+5A5Ugqg3NABtCjaC6ytY0htfXxHDdTxaktvc2Vil1dzQK824T3RCbVjQBDvQM2wOiMu5ELM8dPS9b8cSLeLfRzxzeUjXCC0lc2x82MSm3JtkRtkbTELvuCxRNu8bt+54Ju5k1HWbfUTdi7vr17m1a6tTA9xbx29rH5pUcA/MgP3Tkn5EIKLn2yeK9HsJZtNSS5luNT1VIdPltgsSKXupopHY4bLSLGAxZUKSAbc4c17zXdftpbKKyv8AWZtPuL2OFru70pUuyDBcvMqQmOPIRY4XUhGJYsAJceXUlvN4uutctEUz2kdz5MMt82nJ5jW4/tAqzkqNshVbZsEAI8gygDFDX1/X/F2naDqVz5l9bzWVpdfZJE09JFuWilnRpJyQFTEKQyqQUVmdtqyDEddR4o164j8Jy6hoEu+6XUIbRD5Ibc/2tIJE2uVB53r95fUMOGrLsLrxXe6pb2dvc3Y0uVLkDUb2yEU+wfZtrlDGoEu5rhEVlQFQHKybfnjubzxdc69HY291fWnn3d1DO/8AZqPb2cYin+yyJIQN+7ETsMuN+FJizsanq+seKNT0r+2NPsZIpIXmigtY4EumguIrK68xw20nP2grB0wfK4JWXB1NQufEun3ktnLeX01ifs81xqNpp6vLCHFwHSCMI+5Q8duNpWR1WVizHhlLC/1nVfAttc3kXn6kNbSKZPsqHYkepBG+QFwNiKeQzbdu7cSN1Gg22s2XgTxDbWU19LrKXepG0a8gSImVpZGiZcoqMrFlfdyuXYZAG1ZNR1nxXOQ2kaVICzzXFqs8QRZYBbSBFm3MCjm48o7fkbY69CsoTHm1nxNGmprBd6zJZqlqLG5uNHaOeUs0/m/cgcqQVQbmgA2hRtBdZW6QXWsp4EhuE+3NqDbDM80SNOkRlHmOqLGoLLGWZFMYY4UNHuylZces6/b+B/GOoXUt2s9ik76ZPdWawu0S2qOkhTGCS24nIHOQVTHlrH/afiXypPMl1VLHzYhd3a6YoubZysplSCII++NXW2UMFm4lkPmPt3JXvtW8UldLiWbVVaTfkWtiUmkj81lid2e2kiEjRhGKMYAjE7sBgI49cvvEd1deJLaD+0rqBLK7EFuun4gZhEwRWSa2/eDdgfJNJvbBCBCwSnK/iW30fXrK2tL6Rnl1dlsptNVoGgZrl45VYhvMkaUxqEJ+ZG/1RGJDHdabq/2PVx5OpWkUep29yslrYB5ZF/te6diMxszhUMcoUA44OMOwbsItU1ePxBPp1y8hg055r64njiDtPauG8iIRqC2dxkXK5Y/ZOR+9Fbl/omnapdW9zeW/mzW2PKbey7cSxy9ARn54Yjz/AHcdCQeLtr3xpb6Jpc1zNd3NxfaYk98RYKrWLB7dX8tAufN8qSd9jb9zxfKgGYz0HhrVppXexvri7leR5JLCW9tTBPcW6LEHeRNiBSsspQAqpKhSAeWPN61N4u/4R7UZyZ71rqXVLRLA6cjIkCpcmByu0l2YxxKCfkZZFGwsdxPEMuqX+k+JLK5udc+3S2l/FDpkGmb7SRPLkEOJhCcsybGx5v3ztwPuDsNEvbyXzYtTSdL5pZC0XkEQxKuwBY5AMOpDKwYncxL/ACptaOPDutS8UC/1DS7BY57ywS4vY3kCBbqORG+ywlcjaPMLruDZ/wBEy2BKKr2ur61b3UVwk2s32hw3EZuLi70spdMGinDoIVhR2RX+ykMsecu/zFVYLIB4g/4lupWcG3UtR1CW3uJbmFgq2C/a3tjIgGV2h4zj5GZiEZlJJFe71HxZZeNLeyDzvp6y28Ucj27ulxEQgkkk8q1ZVk3GQD97Eo2qSm3l9DxFP4lj1HWZtMnnS1sNKju7W3jtFk+13QM5MW4gkqQkYZV+blNrJzujgutfufEDWrXOpQvJcXEVxELJRb2tsBJ5M0MzR4eU4gJUvIMyPlBtITPh06+n+AB05DqQvxoTQ+U9tsn3rGQYfLZAcZHlj5clcEEkhqksItdtL7U9Ttbi+lt5dVsQkE1iEe6jkhtIpZpMqGG1dxwgjCsj7sj5Vx9Tm8XaxoerafOdVKz+H7x7mMacihL4eVm2ibbmSPBdVIB3BmKySEZj6TxZFq9zouhTaH9rudQieaSCaaAI4kNhcrG8isqqhLsgIZVALAEDpUeoa3rF5eS3Fh/atroR+zxSzppji5hbFwZGjhkiLvlvsqk+Wy4ZsYIZlr2D6ovitbqCTVZ9NuLSyin1C6sPLuGKz3WEMRjQKuWVWYJlUZTgBjKmpoWqavLevBqTyFNKQWN26RBzd3TuuyXagzGPK8uQgZUC5IJHlGjXfDlppnw78RadoljIDPpkqJDGXkZ2W2EKAAkknbGi4HXHqTnL1688Y6X9ptbWWe9j/wBGmlvUttjRB/PEiQhIpcqrRwAKY5XCync2MOupBqGvSeAYrthIdSLhZXjtnEqQ+dteQRyRozSrDlseWAzj5UIIU17PWbmxuLS5ku/EGoaSUuI53utHdZxPmExARRwI+zaJvm27ckgtnaBHoE/jFrK9k1yf7M1lFGf+PTznmlNnEXGyMDfGspkOIyWdvlBUJh8dNW8Yjw9NcRHVZZDdxxyyzQ4aKLY5JiX7Cshbf5atmCRdrcEEOUr69e+LNS8F3lvLHfPNcaVNHBFa6a7i7cmZJFm8yONkZYhEykCLc7NtWQYjrtPFF3q9tcWS6ebtIGSUl7S1Fwz3AKeTE6ngRMDJuYlANq/vI888nHr/AIuhs9Glv5L6C6ll0uKaJtPQQtDKYFmkdyBsmMryR7ASQuCIh/rADUPF11e6dDcQX08LfZ5b6OfS08u3uheWnmRxtt+aNEMpV8HgMwlkK5j3PDV74jt00STXZru6/tDTBLdj7BsFrcBoFRAEXKlvNcvvJGYywEaggXPGWh29zour6pBYfadXj0+UWw2GXc6wzqi+Wcq//HxKNpU538g4GDXNO1G+8ZaO1ldz2MK6fepNdwwK7LmS2KqpcFFYlT95WyquAM/MvPrrnjG4/syeS3vra6nl04zWcOnboFtpPJM7s7DKSB2lQruJVFz5Y/1ouXd940TRE+zwyNe2FkIL93jUC4uC8IaaDajFgsQmcYQjMiLsd1aNdTwjc6xqnh67XU7ieK4814oZzA6TKpRSGIlt4lZgxbBEW3AUHcQxPJ6Rc+I9O8HaJa27azFPBo8Is4DpuftF4N6tb3OYsxRKVhUMTHlWY72xuXrPFdtcS614TuYpr5IbfVSZltYBKAGglUM/yMVXJCFuABISSCFZePtb7xhqpW11i3nLC7sZ54Ps0xEMiXtuWET/AGaNPLVfMJ/ezHCqQxAZm2LbS9Ufwj8O7WAT2V1beQLiQ2+57YCwmRiVPCtk7QWBAYrkN9014H8VXniGG3aS+so3lW1m1CGwiEjxRPqIVmZ42X5vLtmPG0GX5Qok50NIvbzUbUaT4rSdodc0+AiKaAxYnninae1UqBtVEiyNxLjdyxJWtDxPpLxeGBaaNDsmbVbW5XMbTBXa+jlkkZcglQSzkAjAzyAOOf1vUfGOns9nbvfS2lvdvF/aIt/38w8qB0yI7WZSu6SdSVhUfulBYNkPua3qWsx+CrG6ikkh1OZITN9ksp2O4rlgq+TK8Qz3kibgbSFZgy8/q2q+MLfw+ZLM6z9tGmNLZj7BHO1zOTJ5glxGgQIgiaMERMd2GSRwYqp+IX8Sy+GtdsI7S+RZ4tRW0t7TTVdbsvcXIcS4A2Yi8p0bchdnJ/fH5D2Hiu2uJda8J3MU18kNvqpMy2sAlADQSqGf5GKrkhC3AAkJJBCstfwtf67c6jGuoPfPutGfUI7qzEMdndZTEVu2xfNj5m+bdLxGh3c5bCuP+EosNB1IWK3M09ze6qiWhsAUjQi7khkyynezOsQyTsKui7cnc1+a88UpLKY7q9ZFkuHTdZ8YjeYIhAgyUYCI8EM38Jwa7YT7pWH74dQAYjjjPOce3r6evNK6luw6CCeUKSM5gJP3uR93pitWiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiv/Z",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAJkElEQVR4Ae1YP68f1RHdH3GMFBApohQkEhItBR8AIYUvkCJSSpp0fA5SRenTpUsXiQJKCmQXICEiCBBBlAYSK0CRIvJrQJhlzpmZ+2f3t+/NMzZopFn77b135syZuXN27z77tC7zdVqX07I1KkRc4wRI/p0J+krxLQqOvuCsL3vUdrbBbJZb9G69xT+yRWBTvrGtb15fiToD6KY+m1kf9mq34ZMIfLKsJ05PcsFgC3phcSS9CgWKaC6BcSpzSAxcjNYkZAFK42hkkEaqx93qbWiYPYAZQaKW7mjZ6DjdYMB0W/0pkFFAvlpgV5M/ykTSanZH+9JHVOaSzi8CzEA51QD06ezeoJ0ZIA/w3YDb2N20LGc23J3ycMv7LEReLFy7teFhH7y6VMPkMLgOrEnZGT45bTEmF9MhWjJuawMD8dJP3f5VG0YVEmFgxPNZGdc0ml0GR3v9PjaH4zl6GRoIU6tsmhI83nTbU8C21oFpiDxQWDvPlhmPcnsX/SD3nZtd27H6g2EjQQPNkF4QVjrEWVYszTRM4RF33ySXXDtKC9AkpHSHJTNSJx9L4NzL2Dnux/BAyQIFnM3nxt0pLYTaTHTLrmF6icldKnNfUQmhUJYNl5+qfuQPYR7RTXoSc200jbaD2pOgPs/nj8fBIz3GP9i5J1ZW9N17Pz6yBznled959haDnHeowmiGfxzZEv+a+YeNa1eDXzaLALnYIZBMbI5Bs5pDPPQDgD+28JNAsNgKvJ7DJoYE3kLJbfFKNhES1sEOpIVFiMLs8NRqmCyHtglrvvaOFW+LoA2MctlcBs4w0u7BzQrDlGIMn3kQwzg9eCwQA884zcrylLCXscsqRIePNE9NPxxt2yyKN/XqenOAwjg8TQPSrW1EgdyKEkmcWOi1iSLbPnYwM0yEDvLRnbIW1vWGZJxyOs6yCLy5h9CphhHj4W2ckM3KCV0DqRgbVZuo8UzgbOLKcx1siY/UDdkOdqwt9Q6jFYzGzUuiBQv1YkmXYnTZMFqOIrUYp1GPkQzthL2laxMxHnMiotG2XLodtXe3JTwNFmT8fq/7Tn4UeGQfttVOadh4ROqBaifKgOxTtH+4bLmxCmBvGaIwberskGLY2YbgFjjYJkKJ1niME9l4aKE/gR5tkhwvj8raR+yQO8M+5nKLE2D0OSMeGT546mgf9+kbiG8nOsWHgO3XGUGWuwcIlm3V0WKBcodyYalGczQkHHDpDfc5/wgkhecmUrHAMF7JCJN/HuouRVhqOwhME9buQLg1iz4sMUFduGgEQr2YNdsMahDG6c2jHdm/HY2IYR3dZ8T60Uw4fGRUUCdboLAY+0YU0UymKKyCWU1W7oWBLXSYtOBuM343WFFkRGFnImCbL6ItqXiM0gZAtTqrsQGcRE5xZILClr6TOGa2WGMEzYZ5JpdkCBop1dxA0+6adYjd2Vw5PkpDtQbsxXOLqM7Sj56e4IbEeen7OvHxUr9+5fpKfquDx8ydAWZZwd6SjKDuUKs2z6Fm0yULRm/1B5yen/xYWCZxST71ThhlkrvRyHvd62rOa092JDuDJKRt7xieL817BnJJQddBE8vvsJ2I53i1jdKhc06z7ZMOXWyBUKHLrubJZDks1mpq4Qf5h0xEHOBpJvbcfwCM3FvC0Zdo3rfBDeNFl46iDf1UVoPYxMQG8aZaq9XnfHvcBI6BqgmFaMsAgPzhRTM8nsMMUpEVOZdFoGLkjr+M1HDl0ER0KYDZrKjhfy3lwdQXzZ9QGoTRE/cugVJW7kHlE5gLReirq0eTm6UWMyMSF7DjqCs3DWU5UAEk9DiihFou2Oe7GSVNf6Sn/SBsNrRUOCxnl7JJBu/wzg26Tnjg1n1v6RXcqJVJ7yjDfzOw1I1aJ/v7oPBIdWa+0WSDIHPLtnFulyrD1irr4xxnqT2pbNz9jdqkZpJmxGr8xwNTQohJQT9dpRq1c4AVX0jc7XLgsITPyJxTUATaWhk8FNSkd/TAxUQK1CD6sBmtw38pgIE5xrvxSDEbZnWcNXpM20I3PNjZFdmvlWzPhXe4HZqufRNtIHefmHgkqMtOW3UOEHKSeqCYp2PMGPn8P4F7eQbf72q/E7Sg6bzvR080+drCQtu64Q8dDaETjRzjn31/A3mwSyiMd956bB89E0iMfjxSyrMLKxxYovW8BCcvY2z8WMuP8RkG6Z38g+X00jvL6ZXnf/3ih6ff/GNZLt64uPv1vbuv/+vfH936y7sa8R3uq17yyq9SovzIH0xg8Dt9WOoP7+7H2AJloX9hxNUYGaw3QkANKMM5A5ohT6zrj2T63PrH9/66vrCu33y5frX+T1x/e/v91z8B5Ltc8yktBfDiYciZKWP2cZg8HjgCOFeHY9vx6gl8HOP+vyz3ZP3N8t4nj/7i6eXeV/du/nh5/OL02BNf/uyOM40B15rPG/ZQoZXzG2/W4U4u8ThLHxuLTPi+et0+dmib/Wr53S/ffO5iufOTn4vt5s1lefLrnz7zVPPf5wTv8OaSX15QoO5X3iwrCt+29q9JXRyWu3EgkCaM8rQajY+SbCB/6tU/i+G3F3/4+2d/+u8Hv1/eXm59/Nry+X9uf/HpW7ff+XBT7LWX3Nu1o64KeDisV2UN+c8oHIq7HNQe4cthP4T34Wz4h9hJMGdtONiotLBSOK10wcJL4WCj0sJK4bTSBQsvhYONSgsrhdNKFyy8FA42Ki2sFE4rXbDwUjjYqLSwUjitdMHCS+Fgo9LCSuG00gULL4WDjUoLK4XTShcsvBQONiotrBROK12w8FI42Ki0sFI4rXTBwkvhYKPSwkrhtNIFCy+Fg41KCyuF00oXLLwUDjYqLawUTitdsPBSONiotLBSOK10wcJL4WCj0sJK4bTSBQsvhYONSgsrhdNKFyy8FA42Ki2sFE4rXbDwUjjYqLSwUjitdMHCS+Fgo9LCSuG00gULL4WDjUoLK4XTShcsvBQONiotrBROK12w8FI42Ki0sFI4rXTBwkvhYKPSwkrhtNIFCy+Fg41KCyuF00oXLLwUDjYqLawUTitdsPBSONiotLBSOK10wcJL4WCj0sJK4bTSBQsvhYONSgsrhdNKFyy8FA42Ki2sFE4rXbDwUjjYqLSwUjitdMHCS+Fgo9LCSuG00gULL4WDjUoLK4XTShcsvBQONiotrBROK12w8FI42Ki0sFI4rXTBwkvhYKPSwkrhtNIFCy+Fg41KCyuF00oXLLwUDjYqLawUTitdsPBSONiotLBSOK10wcJL4WCj0sK+BTiMoZyyXFt5AAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen1.to_PIL(binary_method='gray_scale')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 2300\n",
    "txt = data[idx]['text']\n",
    "words = txt.split()\n",
    "prompt = process_lambda(' '.join(words[:-1]))\n",
    "prompt = TGraph.from_text([prompt])\n",
    "prompt = prompt._spacing_text(3)\n",
    "prompt.set_device('cuda')\n",
    "target = words[-1]\n",
    "gen1 = model1.autoregressive_generate(prompt, None, 4, 'gray_scale', temperature=0.1)\n",
    "gen2 = model2.autoregressive_generate(prompt, None, 4, 'gray_scale', temperature=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bastian'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt.to_file('readability/prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APR9fg1238Q6/q2lPOn2bRLeWGKO1En2yeN7plhyQcryAyqA53phl/iy7q+8R28kFrbf2lbh728LRWGn+WzKbyUI29raSElk2kbmi673dg+5djxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdK9vqfit7+zF9Hd2+oM9oDYQWgeyeJkjNw7zbW2ujG42r5q58qP5W3fPl28ni671q2vhHfPfW2lTvIk9kkMa3XnWry2cchwDGwjKLIQ2AzMJJMEJ1EN9rOq+DdX1TTJM3V7FNNoybU+RPLxActgHeVEvzgbfN2n7tU9R1nxXOQ2kaVICzzXFqs8QRZYBbSBFm3MCjm48o7fkbY69CsoTH0jUfGWoS21oby7hikvUSW5msJDLHEYLgvzJawIBuSHaQrbXPzbgwQ6FgmrzeM9BvNQl1ICO31SyOLQeW+y4QRPIwj+QyRoHzlVJRdoAJVs/Uodfg8Y6m+mwXdul1eqv2yKzWRkRv7KjZkZ0YAbBPnt+7JIOzg1TWfGGlWIkilu5pJrhoFM9nHhDHqEFvFgYjBM8Tu53MATyhjUcdJpt7rr+GdXmtknvbqLzP7KfUYBby3WIlK+amI9v70un3Uyqqe+48/b33i6816KytL/VRpMksCJf3ukpHIYzFdmUkFVwwdIVBZEAPl5Rw2Zafie68V3nhjVrB4dSd2sru2tUg08SfbiJJ4mM3A2HyVhkUqYwzSHasnEddZrmnajfeMtHayu57GFdPvUmu4YFdlzJbFVUuCisSp+8rZVXAGfmXm7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJfQ8S6zr8Hiq9sdLluykOmQXFrDBZrKst2zzhI5XwSkT+WAzYAG3PmR/xx3OreKYtejVWvkh+13Qu0/sozW8EKxTm2ZGXDybwsbMqlzvwu6IkI1ODVvEw8NXk73Wsi5NxHHG0lkzkrhiwTbYq6HpuZ4JExhVYMxKR6nrPjW0g01rdb4IbQSiS4t2kaacu+5Zlgs3bywoiwNtu5ViD8+RHoNJrt5420cXUd80Nvqty80f2IC3toxBcJbskvG7ejKW5kAcgExHCNoX9/rqeKGhhe+GLuBLa1SzDWk9qRH50sk2w7JFzPhfMTPlx/K2758PVp/Eeu+CrrTmTUlu4vDlzHqYFjsMt9ti2pGWTEgbbcLmLK4bgglDXaW+qRT6Ldm0u55biyiCSS3Fk5kEnkrIC8KqjMxV0YooU5O0YPA4OY6/qvhG4lvLrxAHg1OwlBW0V22LcIXKRmzikYqPnP7sj5FwzfvFrcutX8UiJrOwtZ5tXt7u+ldJrUpBJb7bg2q+aQsbZJtQQr7+u7GHxn3kuqXEmlz6Rc65rF1bagZNuraZ9lWIG0ul3L+5hDcsBgsAW2KWj3bq3NctL7UI/Cf2C6uw63u+S9ltf3saGznUyOhVQjncANygK7DKn7p5/WNY8aQ6Te/ZxqS3lvb3CWHkaas32ySOadC0wK4U+UkMi4KBmkO1ZBiOrGiaDdnxjpN/cpdpFavrjqj26bEaS9+Q7mTcpdHJBDDIQY4L7rGrJq9j4z1u8spdSYT6ZYhdloJI441uJBOY2EZ3SxxsXVCWLGQ/K4CqNTTb3XX8M6vNbJPe3UXmf2U+owC3lusRKV81MR7f3pdPuplVU99x4+4u/FMGo6pfaaNVuLeSKxgF/d2BguDCpu2YhFhclhIyL/AMe+djA7cYmNi/1vxqPDjzwfbhfrp8jWS2umNILmYPKGEwlijKMsawsp2xB3ZtqSDEdSSaTq7+MrtLe71JC3iNbgXElkGjhhbTHQvG2wKTnMQJLBSse5SSd/SfbddXwn55Sc31tqHlyN5A8yW1iu9jvsA+Znt1LfIvJb5AMqKy9SvfFF5c2UUmlSRRR3ET3Ef2dJU2tfWxhIPzfOkIn3lT8jIW4HlMcvQNE1k6xawLqOq23ky60s95NYoHAe7hdBGWjEf7wDfu2uDmQLjA2dJLd+I9R0Pwo8Rk068v3jOplLXd9nVrWR3G187CJAoBbOG2ghhlW5vVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNlh9Z8cJ4j1GGG23sPtQtrWWGUwOFRzblXFuqKzFYixa5I+ZxhWIVMuWPxHL4pOpafa6lqLwJEbC41Gw+zGZltdSwJAAgAEsipkrHwycch3uW2u+Kzod/5txfCRZYRC5065ac537wG+wKqr8qEHyJMfMCQXQr3Hhu6uJ9HtkvftxvPKEsjXkQViGZscrGin7vTarhdu9VY4ri9HHivRPCehWqtdyiXR4TcKLIBtPEbWyERrgsZRDJM2195Z4vlQDKHQ0XVfFEd1c3d/BqV7pkVxLbwxGySG4lVorZoJCvHBbzlY5XDSZZY1VhFTTUfHFl4j06yv33qv2WKWSK3leC4BRBNI3l2rbW3mULmWIDahZNuS+x4Un8S/8SU61PPctf6Ubi7ElosK2s6+SFQAAFWYSSFgxOWQlQgBWs+3fVNF8T65cmTVZrFtbjmul+weYDbNY7VMWyPdJiYRodm4qI1LYy7Nc0O48V6hrkyXckltpexblJJLcLIyG6uSkagrwWhFvv3fMgGNody65a3/AI5tfDlhOXnvb+90pbi5V7MRCzkV7cMFCoxEnlyTMVZZCXi+SPGYyWWseLJdOgM73wUSzLHLFpzyTSygRGGKVZIIR5bFpsuqRKAiAyqdxbQszqmmfDfwrAi31nstLaLUJLa1825tkFufuxFHJbzBGhGxsBmOBjcsfha78aalqDR6uZLO3RBLK0lqsblhd3Q8tB8y4aNIdx3vhQNpJk8wV7bVfFZijlzqUsqJDNqsU1gEW0cTw+bFbfuwZkMRuRlTKTsTa2WG+5Fc+Jda8RlLe8vtO0Sf7RJFMdPVJQiJaBABKmY2MhuceYp3LuwMbGXHur7xTrEGlfbrO+imlu9KnksodPJgSHfbySys5+ZJFm8xCu5iEXJjxmUbHjjUfEtl9t/sX7csiaeZLBLSxW5W4n/eeYJcj5diiNk+ZdzMQBKR5dRvd+NLe3e9tzJeXM17qFtBYS2qxxRxILlrd3bhss0cKhyyoUkHG47zY8Ii+l8Va1dXDalPaNZWcVrd6hZfZpJgr3JYFcLyrPj7iHG35SCHfLsdW8Uzrqk3narKqbNr/YTEkcbSqJHjilto5PMWLeyIGuB8uG3Hbvk07VfGhurvEF3cws9xb2DXtksAZjFatbyTBcFU/wCPlnPy87l2q+yIaGqvq+r+B/GYmsrtfPt7iPT7V4h5u37KqlNq53HzvNAPO7gqSpUmn/afiXypPMl1VLHzYhd3a6YoubZysplSCII++NXW2UMFm4lkPmPt3JntrHjGW201Ll9Vhu5JbgTw2mneW4H2mRU+ZoJ4fuKu0NJGB95pHVwyyXMOv2HiHW5NMgu767bWGubdLuzXyMHS3EZWXYoA81ViJD5AVQSC7F5Le+8XXmvRWVpf6qNJklgRL+90lI5DGYrsykgquGDpCoLIgB8vKOGzL0H23XV8J+eUnN9bah5cjeQPMltYrvY77APmZ7dS3yLyW+QDKis++1Xxc7/JYT2ixxSNc+XCkhSN7iHy3TBffMlv9oyi7h5ifdZWj38ver4p1TVI55LK+vPst2EsnurM22+EXOlyIZNinbnbKzHbkbX+RNpRdxZPFFz4gs0t5Lu2F0lvHe3kmlorR4F+ZUBwQAjCEISzqNyHMgcmSnr+v+LtO0HUrnzL63msrS6+ySJp6SLctFLOjSTkgKmIUhlUgorM7bVkGI66jxXbXEuteE7mKa+SG31UmZbWASgBoJVDP8jFVyQhbgASEkghWXDttV8VmKOXOpSyokM2qxTWARbRxPD5sVt+7BmQxG5GVMpOxNrZYb7l9qvi53+SwntFjika58uFJCkb3EPlumC++ZLf7RlF3DzE+6ytHvp6Re+NNQ1t7NpruLT3e5Vby7sFhdYQloYpAu0gytvnwG2jJcsg8vyRI0mu3njbRxdR3zQ2+q3LzR/YgLe2jEFwluyS8bt6MpbmQByATEcI1jyNdsde13UbV5xbvrdoqWi2oK3EbxWsUsjMQWKou4gptCtG24sPlGHYa74+cSrLbyG8L25ktpIJQkbm5hWREk+yqgi2NKGbfOQAGViFLHuNTOr6f4dR4rmS7vYriCS4kitxlofPVpgkYBOBFvCqNz4AGWfk8/qV74ovLmyik0qSKKO4ie4j+zpKm1r62MJB+b50hE+8qfkZC3A8pjl6BomsnWLWBdR1W28mXWlnvJrFA4D3cLoIy0Yj/eAb921wcyBcYGzcvU1rXNK8E3RSSw1CS4Se8ZLYn7IWspw/ytnaQzbRvyAxXIb7px7jUfE0A1KVku4Z1SG2juU0xmluFjubxc744JcExrE5JjZBvO1U80MNjUYtX1D4f2AH2t9Ta9sZN13AHdcXkTb2SNY8oFG7BWNto+YI27Fe6vfFFpdS2XnalLb21xJHb3qWCSy3cnlQPEkihVTyi0lwrOPKA8pQZEOS1fQr3xpLpsOo6hNd+ZHcadEbFrBU8xZYrYXDv8u7CNJMw27drI24so2rni88TNqUWoXH9pLcQ6PcJLI+lMVtrt5bXfBCY0Yug2/I/lyActumCsq9Jpc/iK58A6nNG98+ttFP9ja8SNSZNmI9oMMPy7v+ekanO7qu1jl674l1qTVrOTS31K20m5uIbYltMInyIbuScpFKquxCrAwIDDK4VZDuRus8OXt5daYiagk4ul3MHlgMZkhMjrC7cACRkVWZAAVLcqmQK5PXNEufDV7bS+EtNkUx6ZPGzAu4QB7CLIJD/OIISVG1yTHnY5yDXtZNe1I+GbzUbjUh9k11lJisXLmJrZwGm32sZxvYpvVFQLJydw3jc1+TxF/b8/8AZ91fRWccVhGkcNtG6M01xJHO+5kYlo49jjB2rgFgVJB5/Vdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNnQeOr/AMQWX2AaOs6Wr+Z9omtkZ5FcbfLXC29wdpBkJPl9VX51+6xaaj4gmnsjP54tZfsrXksViy+VKyAtHEsgDmEts3MyFo97DP3jb8vpeo+JfEHh/RZbv7deef8A2bdXnmWKxJC4u7V0aEoMOrR+bI5DNt2jIi+5Vgah4uur3TobiC+nhb7PLfRz6Wnl290Ly08yONtvzRohlKvg8BmEshXMexoOqa7p8Gh3HiS7nlXUdPRrrzLIRLa3TPbxxRgKuVZzM24OT8yEqEUFRc13w5aaZ8O/EWnaJYyAz6ZKiQxl5GdlthCgAJJJ2xouB1x6k5y9evPGOl/abW1lnvY/9GmlvUttjRB/PEiQhIpcqrRwAKY5XCync2MOupDqWst8PDezySJqmxlDpZTs/wDrCqnyzCH3lcZbydoJL+WUG049tqXij+y7KR5NZ/ePOfNNkkzyTDyxDEU8mErA2ZdzSJEQV/1iqVZqek6h4uFlHNdwX1tNP5cskkGloZJ7r7HZ7Y5F2j92zm4DOSm0xqvmxgAV1Hhawivvhvpui6laTqo0qCyvIJo3iPNugdcnB6NglehyMgggV9X/ALR0K6XT9G/dw6xvitW+U/ZLx5Xlllw2d+UeWXacKPs20cyAVn39/wCLnn1GHTXvjqZ+2Itq9mi2kEYSX7NLHMyBXkZhb5UyOB5kmVXadnSaPqdzqus308cd2mkrbwJALq0e3bz90plwsiq5G0w8429cchq830ybxfo/hmwt7axu7OAJAk0kVoyzhlsbQKpC207MN3ngsY+DGF3rgK2xqN14m1I6fHdw3Zdr3S5Xt7XT2+zGASW8kkxdwHRxKJF2MSwRQTEOZBseJ4tXg8Y6fqOlfa5p00fUI7e3EAa3a4/dNGkj7coHK9S6gmJQCMsG0PC91fXFxerJc6leaeiRGC51Oy+yzmUl/MTZ5ceUCiIhtnJdhuOMLz+kXXjCO106e7udSuJPs+mTzxT2Uah3uZfLuY2CxqQIVXeACCpYlyy4APC3iPxDHqUn/CUmS3idESSN7eYrFcvLHGkcTi2RNm6Qr/rJiflIYgMx6y30my8N6Ldx6RD9mVYgw/dyXODHCsSHYDufCRINqkFsepzXH6dqfiV9JeW6uNcWE3aR3FyLNZJYIvLc5hjNpE7MZPKVswuoVsqchylew1DxczzXOqQX1n5myO7u7bS0NxFGtxf7fLQLJvxi2G396AkhYH5jJVyCx1qb4bRRPayQas+uidkktSdhOqeZvMaufkC/Pw5G3kOR81WLq98UWl1LZedqUtvbXEkdvepYJLLdyeVA8SSKFVPKLSXCs48oDylBkQ5LSaLe+LptJ0+C+ScX959huvPeBESGERwm6jkGBtkLLMAuM/vl28K5jw7DWvH04lilEkF472/mJJYyypbubmFZAh+zxIYgjS7v30pIUFXwC57jQri+jur/AE3UJLu4NvcFba7nt9pni8qFyzMirHkPKyjAGQh4JVjXn95H4j/4R/xHpcNrqTi4fWR9jawxEbdzctHKkgBLytKUULuBKNkRkYkPSXWr+KRE1nYWs82r293fSuk1qUgkt9twbVfNIWNsk2oIV9/XdjD4seBdR1u++3pq7zyRx+WYWubeWOTJ3bhlrW3UrwuAqkglsnBUDl/DT+JdL0KythaX1lGsUCqYNNV5JJ1sbMRxSKQMxlvPVnYptMaqZYwBVjwnP4ll0s+IDPPI17d6ZI9utou26E1tZxzzMcZ2qC5GzaFZG3bh8o6TwXY3cHh3U7Ge81JbganfgT3FukbpundldMxhHBDCTOGXLEdBtHH+HNN1fw7pemxxQ6lZ276PpqX1xb2AeeD/AI/JHCRiMhnErxKw2MwVyzc5cXG1jxjLbaaly+qw3cktwJ4bTTvLcD7TIqfM0E8P3FXaGkjA+80jq4ZblsnivR7CWbTUkuZbjU9VSHT5bYLEil7qaKR2OGy0ixgMWVCkgG3OHNfTdZ8VS2Uy3c+pJaLcRCa6j0+WS7hjKS5KK9nErjesC4WKQqHckgbSupoza7ZfDuSRrae31NtQuZJ1FuPMjje9cySJHlwWEbM6KDICQuPMyA2fb6r4ldp11G61y10xbuRYb230hWunQRW5hBiEcnyvuuGZtgwygHyv9XWpbXHiuaPW5rSSSaKzQNpiXluIpb5zZxFVkBVAieazk4wS5K5jEZVs+51bXotLvTY32sz24eAQXV3pLx3TSHzPNjCpanYgVYiJGgcEs65yQUjvdT8SrBo0UVxrnmSxbp1js1EzFnIBMhtGh5GNquLcqMGTBY7Og1zTtRvvGWjtZXc9jCun3qTXcMCuy5ktiqqXBRWJU/eVsqrgDPzLzdpr/ja41vSXvLSSziuEs2e3FrMYisiRmbeFt3KOrtKo3Tx7diFlxkv1ng5tduPD1je+IJcX09pCXthGE8pgnzFvlB8xiSWGAq8KB8pZ8O4l8UXFnpcNxbySSWT2cOpF7ZHW5uPtVtmaIbT8ixrM+4BQvmL0dGEefa33iPUJJzP/AGldxre2ZVZdP8uAKbyLcwjlto5FIQMww82wAlnDBGaRNT8YyW0Qtpb430n2cX63WmfubGdrmBWSHCJ50Oxrjcwd8LGp8xc7m6zQri+jur/TdQku7g29wVtrue32meLyoXLMyKseQ8rKMAZCHglWNU7rQ7fTNe0a50yww1xqDm8k2GXahivJd2Wz5f72dvmGP9Ztzg4rD1GXxHp994wl0+3u4kkeaexkgtvOa4uF0+EIpBU4TcrYIBDOu0lcBZbmoXPiXT7yWzlvL6axP2ea41G009XlhDi4DpBGEfcoeO3G0rI6rKxZjwyljqvi5H+ewnu1kija28yFIy8aXE3mPJkpsme3+z4Rto8x/uqqybLksur3XhSCaa3kluZNYhdEktgWW1N+pQlCvylYNpyQGTGThgSMv4jya7Np2r2FjHfPDNpTpbQ2lkJ1upGEizLKf4NqeWU+ZCWY4EpGysfxC/iWXw1rthHaXyLPFqK2lvaaarrdl7i5DiXAGzEXlOjbkLs5P74/IfQILHRtX1FNchj826glaFZtzrteEzwkbTgHBlnHTndnnCkYeo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz4Y1DxddXunQ3EF9PC32eW+jn0tPLt7oXlp5kcbbfmjRDKVfB4DMJZCuY9Dw3Jrt74y0+61WO+Pl6VcrcmeyEUdvdPJbF4I3GN8Y2fKSGyAxEknOz5Aor7/ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooor/9k=",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAJRklEQVR4Ae1Yi5LjOA5Lpub/f7mPAAg9HKcn7NutG14pO21LJAhBhGN17/Pr8XjGj68YZYDD2/nMaGQC3a/ZbU7IB7it6EXQi8QLo6e+gy3Gz8fX49dg5q7H7EeDf4DiR+t+XBQCteHnMzoQ/+KjYt410zDCTCkvLEKqdBUYHHIsAolj5oXfLMiKd71i3WSKATJUQiipIpgrMAEQ8VnECRFc5/H4jSq4HZufDwADCmFI5nEhTElVfjGiPJlQI0revQJiQjEbl1zS08giuF6fIFfVxD8eY0nhxzTrzcDlBwNR3DAXAUv8UFEGpA7xwIo6Q8F4AxxJ8wgzaLVEBMUvijFlVhpSiQGYvvkk8pq1WG9kMnDDhkdaSOOV2JfLrZvKtZf7wrCWr+PcdhQajKw0jCtpt6rLQkPzFr+3B5B1w7leRKMhX2k4hpEwA+76AQwM+FixZriSIe/Wm1QGgVYUBCurmK4CKC6qoQpLclFzPL+AlmiQWi8WGwyYvEpF9ObzMZC1JXQJPKX9rAxvaXRp/VzmMWXzLhCF1+AypvfzVbpkLuTIjAdFsBvEbf2lbMG8bolJEs9zeCv40+S7xf5U+7/Nx4Z9/ElInmLz/JJPCufDsBx8siMCGOgI9NV8mRGSKF2SOMuSQnJYgsW4UhK4FLyxHMsBwdKEICQsIPFJ1BaMlxa+3/g+8DuBsYY+Afn1TwhZ5O4eZh4Eepe4dmGigqyfaROKHwTb60aML6SQip3xbuVj7qRRmHOPQGxvaQb4UhNdzPmixD1zl9sIa0ARl1rG9AZVsWtMHbKZcHxZAvvNpYWZgqiM+QU/ZA7ubWFyrMdSliLu1b3OZJWFc76OVLXXjthVXVBH3xF1wUrF8ZLwcAhaVb4UbihnFeSGfYhFKlqifzH0YJEKgRQZ0CUsTrSTVUutYlK3FCS19qusOHBd5Dg4qDkYEiONgH9nEDq5mRJcMI0XESb//76/O5bQ9csnQjfRFcQX5x3qD3Xi/Q6ktQdCgzG9WxK6FsCQ+W7DA7AN/E3agi+Tz1AvZd8Fbilvg9+xIPcL51h0Io8shLIvPr2YzPQEMmusw6iOcv4HFlZlCOtc8QoklfAoynAEUktSGsGEa13Ne1zWxVkgQMCV+q0vcVwRiqDbhgjeKSBmTO8qRRBSNLMZHgTMjyB5goQlvnA94nJpkSLtn1URlxdcq2ygUcuNXaWTJ8l+fbGjMfNOM+tp3AdEKWScTTBuCkHGmDjI0PUC4I5OBnmlyaVoBHMwlTmD+zKeADHh/2l98Q+rWByhMFUZd513Q5QyImeuUNizC0tO4zZWAD6lbYyxWIQteuInxTLalYF+SXKF3F2Gn4/fWjd08t88ZTHN804QaA0h+vPYtHjQ9IMMPnz0lF6wPlpf8a+MIBGB2MyYj4NTuFtZxpYFk3YAoI7yb1hWxv96/G/z1wSGmnEs6f2W9TLrAzIDfUfTOXZgeDXJlEpAwmfWoz2x1Riy3Xc8U0tRrhZqxoa36vLkZl9vOT7FfoqrLfQr944vLCrzFGQUY70ydZaNrHzMIED6j0XJA6zpkjfmiVUOYS5L4EbOFPEi4VwPjw+VqzivG/HbhVLO47dfCckZu8YLRt3lYau/aBjBhVkN+BaLIUQhwbl4cpo4zhJiqHHjPrAjkvxjBciaqKs4lSGvxdf7lPO8/nko8Soe1xHUQP1QT4DR+3nOWaepK/Mlaey8q94dZuVyWcuX8DY0RsF3C1nd13XDTmycI5i7oD71Erh9ybeVLDDW9w39MvHC36GNyeKYfrfQE79aTukyyzVenw6CV4OAY8cxyRjDAi9UCIxKKFYJRzzMJ5YNBJYQ0pJO5XHlIEI+zpnVRZjBdbfQQJMJon/yGUtsxffRDXI3+WHZpPqc4P2xhF7dPK4+7+LB0Hq+MbG0L+MC3TGNDJzdPtfSmXTGd2eCQEdFBK45Y3B//uQcvqgz35uw0//M3Yv4fmF9E56o+E1LjkVXdH4tR2QO0TCnWckJ+8gTTx1N43nQgopVuowhqhcAeHLZrMg6wTIlFKEBV4HviHIlMMf3H2l+AEgJrlcu/nhIiA+4+DL4+7AMgzma5wQGoCYgTxTdlEgc0xj77+osmUREZF7JVLOUTrRyIMRHxNKgyLyOXA5WjrHh7M7yUCxDLDD5MMLLcQdknm9C5pR2Hd+mrhTW2s02OdfSiWLVBGkqqu3KiltxAYsNhzmAvCD2lS5p72NbCRMkAjuKXTcKxmBdURVrnWjG1Sp38kjffPKhWzLaIQNwWPtNpYhKLbxiFovQIItHZuoeewMT/i1MWTcKNIgrhQ92zWc0EiQzSlMubxA4rGdRkJWZEQj7G024YlnwyeXHhZ+Q/4sY67bpmHu8LCsYk0sUQ3f5Ev5rp/MXj2+Vf5v8azd3J4wb1pEV1uILAtO0Px2LEeTXMpN3JJ1ieGnFk8qd+p76x2N8l+y0x01rPtJ+ZH0HZvw/XQd93wi6TfIXDz3LYfaqPzZIlx30fcW0G+sczuNSp6iNxP54AOvY3M7edvscgvVFHdMYvEbW7P4MRMbt2UB/8eQP2/uLlf9Q2jyHf0jQrexsuJtjVb3H4WrHuuGPw90cq+o9Dlc71g1/HO7mWFXvcbjasW7443A3x6p6j8PVjnXDH4e7OVbVexyudqwb/jjczbGq3uNwtWPd8Mfhbo5V9R6Hqx3rhj8Od3Osqvc4XO1YN/xxuJtjVb3H4WrHuuGPw90cq+o9Dlc71g1/HO7mWFXvcbjasW7443A3x6p6j8PVjnXDH4e7OVbVexyudqwb/jjczbGq3uNwtWPd8Mfhbo5V9R6Hqx3rhj8Od3Osqvc4XO1YN/xxuJtjVb3H4WrHuuGPw90cq+o9Dlc71g1/HO7mWFXvcbjasW7443A3x6p6j8PVjnXDH4e7OVbVexyudqwb/jjczbGq3uNwtWPd8Mfhbo5V9R6Hqx3rhj8Od3Osqvc4XO1YN/xxuJtjVb3H4WrHuuGPw90cq+o9Dlc71g1/HO7mWFXvcbjasW7443A3x6p6j8PVjnXDH4e7OVbVexyudqwb/jjczbGq3uNwtWPd8Mfhbo5V9R6Hqx3rhj8Od3Osqvc4XO1YN/xxuJtjVb3H4WrHuuGPw90cq+o9Dlc71g1/HO7mWFXvcbjasW7443A3x6p6/wPUzCle8m5/WQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt.to_PIL()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen1.to_file('readability/gen1', binary_method='gray_scale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APR9fg1238Q6/q2lPOn2bRLeWGKO1En2yeN7plhyQcryAyqA53phl/iy7q+8R28kFrbf2lbh728LRWGn+WzKbyUI29raSElk2kbmi673dg+5djxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdK9vqfit7+zF9Hd2+oM9oDYQWgeyeJkjNw7zbW2ujG42r5q58qP5W3fPl28ni671q2vhHfPfW2lTvIk9kkMa3XnWry2cchwDGwjKLIQ2AzMJJMEJ1EN9rOq+DdX1TTJM3V7FNNoybU+RPLxActgHeVEvzgbfN2n7tU9R1nxXOQ2kaVICzzXFqs8QRZYBbSBFm3MCjm48o7fkbY69CsoTH0jUfGWoS21oby7hikvUSW5msJDLHEYLgvzJawIBuSHaQrbXPzbgwQ6FgmrzeM9BvNQl1ICO31SyOLQeW+y4QRPIwj+QyRoHzlVJRdoAJVs/Uodfg8Y6m+mwXdul1eqv2yKzWRkRv7KjZkZ0YAbBPnt+7JIOzg1TWfGGlWIkilu5pJrhoFM9nHhDHqEFvFgYjBM8Tu53MATyhjUcdJpt7rr+GdXmtknvbqLzP7KfUYBby3WIlK+amI9v70un3Uyqqe+48/b33i6816KytL/VRpMksCJf3ukpHIYzFdmUkFVwwdIVBZEAPl5Rw2Zafie68V3nhjVrB4dSd2sru2tUg08SfbiJJ4mM3A2HyVhkUqYwzSHasnEddZrmnajfeMtHayu57GFdPvUmu4YFdlzJbFVUuCisSp+8rZVXAGfmXm7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJfQ8S6zr8Hiq9sdLluykOmQXFrDBZrKst2zzhI5XwSkT+WAzYAG3PmR/xx3OreKYtejVWvkh+13Qu0/sozW8EKxTm2ZGXDybwsbMqlzvwu6IkI1ODVvEw8NXk73Wsi5NxHHG0lkzkrhiwTbYq6HpuZ4JExhVYMxKR6nrPjW0g01rdb4IbQSiS4t2kaacu+5Zlgs3bywoiwNtu5ViD8+RHoNJrt5420cXUd80Nvqty80f2IC3toxBcJbskvG7ejKW5kAcgExHCNoX9/rqeKGhhe+GLuBLa1SzDWk9qRH50sk2w7JFzPhfMTPlx/K2758PVp/Eeu+CrrTmTUlu4vDlzHqYFjsMt9ti2pGWTEgbbcLmLK4bgglDXaW+qRT6Ldm0u55biyiCSS3Fk5kEnkrIC8KqjMxV0YooU5O0YPA4OY6/qvhG4lvLrxAHg1OwlBW0V22LcIXKRmzikYqPnP7sj5FwzfvFrcutX8UiJrOwtZ5tXt7u+ldJrUpBJb7bg2q+aQsbZJtQQr7+u7GHxn3kuqXEmlz6Rc65rF1bagZNuraZ9lWIG0ul3L+5hDcsBgsAW2KWj3bq3NctL7UI/Cf2C6uw63u+S9ltf3saGznUyOhVQjncANygK7DKn7p5/WNY8aQ6Te/ZxqS3lvb3CWHkaas32ySOadC0wK4U+UkMi4KBmkO1ZBiOrGiaDdnxjpN/cpdpFavrjqj26bEaS9+Q7mTcpdHJBDDIQY4L7rGrJq9j4z1u8spdSYT6ZYhdloJI441uJBOY2EZ3SxxsXVCWLGQ/K4CqNTTb3XX8M6vNbJPe3UXmf2U+owC3lusRKV81MR7f3pdPuplVU99x4+4u/FMGo6pfaaNVuLeSKxgF/d2BguDCpu2YhFhclhIyL/AMe+djA7cYmNi/1vxqPDjzwfbhfrp8jWS2umNILmYPKGEwlijKMsawsp2xB3ZtqSDEdSSaTq7+MrtLe71JC3iNbgXElkGjhhbTHQvG2wKTnMQJLBSse5SSd/SfbddXwn55Sc31tqHlyN5A8yW1iu9jvsA+Znt1LfIvJb5AMqKy9SvfFF5c2UUmlSRRR3ET3Ef2dJU2tfWxhIPzfOkIn3lT8jIW4HlMcvQNE1k6xawLqOq23ky60s95NYoHAe7hdBGWjEf7wDfu2uDmQLjA2dJLd+I9R0Pwo8Rk068v3jOplLXd9nVrWR3G187CJAoBbOG2ghhlW5vVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNlh9Z8cJ4j1GGG23sPtQtrWWGUwOFRzblXFuqKzFYixa5I+ZxhWIVMuWPxHL4pOpafa6lqLwJEbC41Gw+zGZltdSwJAAgAEsipkrHwycch3uW2u+Kzod/5txfCRZYRC5065ac537wG+wKqr8qEHyJMfMCQXQr3Hhu6uJ9HtkvftxvPKEsjXkQViGZscrGin7vTarhdu9VY4ri9HHivRPCehWqtdyiXR4TcKLIBtPEbWyERrgsZRDJM2195Z4vlQDKHQ0XVfFEd1c3d/BqV7pkVxLbwxGySG4lVorZoJCvHBbzlY5XDSZZY1VhFTTUfHFl4j06yv33qv2WKWSK3leC4BRBNI3l2rbW3mULmWIDahZNuS+x4Un8S/8SU61PPctf6Ubi7ElosK2s6+SFQAAFWYSSFgxOWQlQgBWs+3fVNF8T65cmTVZrFtbjmul+weYDbNY7VMWyPdJiYRodm4qI1LYy7Nc0O48V6hrkyXckltpexblJJLcLIyG6uSkagrwWhFvv3fMgGNody65a3/AI5tfDlhOXnvb+90pbi5V7MRCzkV7cMFCoxEnlyTMVZZCXi+SPGYyWWseLJdOgM73wUSzLHLFpzyTSygRGGKVZIIR5bFpsuqRKAiAyqdxbQszqmmfDfwrAi31nstLaLUJLa1825tkFufuxFHJbzBGhGxsBmOBjcsfha78aalqDR6uZLO3RBLK0lqsblhd3Q8tB8y4aNIdx3vhQNpJk8wV7bVfFZijlzqUsqJDNqsU1gEW0cTw+bFbfuwZkMRuRlTKTsTa2WG+5Fc+Jda8RlLe8vtO0Sf7RJFMdPVJQiJaBABKmY2MhuceYp3LuwMbGXHur7xTrEGlfbrO+imlu9KnksodPJgSHfbySys5+ZJFm8xCu5iEXJjxmUbHjjUfEtl9t/sX7csiaeZLBLSxW5W4n/eeYJcj5diiNk+ZdzMQBKR5dRvd+NLe3e9tzJeXM17qFtBYS2qxxRxILlrd3bhss0cKhyyoUkHG47zY8Ii+l8Va1dXDalPaNZWcVrd6hZfZpJgr3JYFcLyrPj7iHG35SCHfLsdW8Uzrqk3narKqbNr/YTEkcbSqJHjilto5PMWLeyIGuB8uG3Hbvk07VfGhurvEF3cws9xb2DXtksAZjFatbyTBcFU/wCPlnPy87l2q+yIaGqvq+r+B/GYmsrtfPt7iPT7V4h5u37KqlNq53HzvNAPO7gqSpUmn/afiXypPMl1VLHzYhd3a6YoubZysplSCII++NXW2UMFm4lkPmPt3JntrHjGW201Ll9Vhu5JbgTw2mneW4H2mRU+ZoJ4fuKu0NJGB95pHVwyyXMOv2HiHW5NMgu767bWGubdLuzXyMHS3EZWXYoA81ViJD5AVQSC7F5Le+8XXmvRWVpf6qNJklgRL+90lI5DGYrsykgquGDpCoLIgB8vKOGzL0H23XV8J+eUnN9bah5cjeQPMltYrvY77APmZ7dS3yLyW+QDKis++1Xxc7/JYT2ixxSNc+XCkhSN7iHy3TBffMlv9oyi7h5ifdZWj38ver4p1TVI55LK+vPst2EsnurM22+EXOlyIZNinbnbKzHbkbX+RNpRdxZPFFz4gs0t5Lu2F0lvHe3kmlorR4F+ZUBwQAjCEISzqNyHMgcmSnr+v+LtO0HUrnzL63msrS6+ySJp6SLctFLOjSTkgKmIUhlUgorM7bVkGI66jxXbXEuteE7mKa+SG31UmZbWASgBoJVDP8jFVyQhbgASEkghWXDttV8VmKOXOpSyokM2qxTWARbRxPD5sVt+7BmQxG5GVMpOxNrZYb7l9qvi53+SwntFjika58uFJCkb3EPlumC++ZLf7RlF3DzE+6ytHvp6Re+NNQ1t7NpruLT3e5Vby7sFhdYQloYpAu0gytvnwG2jJcsg8vyRI0mu3njbRxdR3zQ2+q3LzR/YgLe2jEFwluyS8bt6MpbmQByATEcI1jyNdsde13UbV5xbvrdoqWi2oK3EbxWsUsjMQWKou4gptCtG24sPlGHYa74+cSrLbyG8L25ktpIJQkbm5hWREk+yqgi2NKGbfOQAGViFLHuNTOr6f4dR4rmS7vYriCS4kitxlofPVpgkYBOBFvCqNz4AGWfk8/qV74ovLmyik0qSKKO4ie4j+zpKm1r62MJB+b50hE+8qfkZC3A8pjl6BomsnWLWBdR1W28mXWlnvJrFA4D3cLoIy0Yj/eAb921wcyBcYGzcvU1rXNK8E3RSSw1CS4Se8ZLYn7IWspw/ytnaQzbRvyAxXIb7px7jUfE0A1KVku4Z1SG2juU0xmluFjubxc744JcExrE5JjZBvO1U80MNjUYtX1D4f2AH2t9Ta9sZN13AHdcXkTb2SNY8oFG7BWNto+YI27Fe6vfFFpdS2XnalLb21xJHb3qWCSy3cnlQPEkihVTyi0lwrOPKA8pQZEOS1fQr3xpLpsOo6hNd+ZHcadEbFrBU8xZYrYXDv8u7CNJMw27drI24so2rni88TNqUWoXH9pLcQ6PcJLI+lMVtrt5bXfBCY0Yug2/I/lyActumCsq9Jpc/iK58A6nNG98+ttFP9ja8SNSZNmI9oMMPy7v+ekanO7qu1jl674l1qTVrOTS31K20m5uIbYltMInyIbuScpFKquxCrAwIDDK4VZDuRus8OXt5daYiagk4ul3MHlgMZkhMjrC7cACRkVWZAAVLcqmQK5PXNEufDV7bS+EtNkUx6ZPGzAu4QB7CLIJD/OIISVG1yTHnY5yDXtZNe1I+GbzUbjUh9k11lJisXLmJrZwGm32sZxvYpvVFQLJydw3jc1+TxF/b8/8AZ91fRWccVhGkcNtG6M01xJHO+5kYlo49jjB2rgFgVJB5/Vdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNnQeOr/AMQWX2AaOs6Wr+Z9omtkZ5FcbfLXC29wdpBkJPl9VX51+6xaaj4gmnsjP54tZfsrXksViy+VKyAtHEsgDmEts3MyFo97DP3jb8vpeo+JfEHh/RZbv7deef8A2bdXnmWKxJC4u7V0aEoMOrR+bI5DNt2jIi+5Vgah4uur3TobiC+nhb7PLfRz6Wnl290Ly08yONtvzRohlKvg8BmEshXMexoOqa7p8Gh3HiS7nlXUdPRrrzLIRLa3TPbxxRgKuVZzM24OT8yEqEUFRc13w5aaZ8O/EWnaJYyAz6ZKiQxl5GdlthCgAJJJ2xouB1x6k5y9evPGOl/abW1lnvY/9GmlvUttjRB/PEiQhIpcqrRwAKY5XCync2MOupDqWst8PDezySJqmxlDpZTs/wDrCqnyzCH3lcZbydoJL+WUG049tqXij+y7KR5NZ/ePOfNNkkzyTDyxDEU8mErA2ZdzSJEQV/1iqVZqek6h4uFlHNdwX1tNP5cskkGloZJ7r7HZ7Y5F2j92zm4DOSm0xqvmxgAV1Hhawivvhvpui6laTqo0qCyvIJo3iPNugdcnB6NglehyMgggV9X/ALR0K6XT9G/dw6xvitW+U/ZLx5Xlllw2d+UeWXacKPs20cyAVn39/wCLnn1GHTXvjqZ+2Itq9mi2kEYSX7NLHMyBXkZhb5UyOB5kmVXadnSaPqdzqus308cd2mkrbwJALq0e3bz90plwsiq5G0w8429cchq830ybxfo/hmwt7axu7OAJAk0kVoyzhlsbQKpC207MN3ngsY+DGF3rgK2xqN14m1I6fHdw3Zdr3S5Xt7XT2+zGASW8kkxdwHRxKJF2MSwRQTEOZBseJ4tXg8Y6fqOlfa5p00fUI7e3EAa3a4/dNGkj7coHK9S6gmJQCMsG0PC91fXFxerJc6leaeiRGC51Oy+yzmUl/MTZ5ceUCiIhtnJdhuOMLz+kXXjCO106e7udSuJPs+mTzxT2Uah3uZfLuY2CxqQIVXeACCpYlyy4APC3iPxDHqUn/CUmS3idESSN7eYrFcvLHGkcTi2RNm6Qr/rJiflIYgMx6y30my8N6Ldx6RD9mVYgw/dyXODHCsSHYDufCRINqkFsepzXH6dqfiV9JeW6uNcWE3aR3FyLNZJYIvLc5hjNpE7MZPKVswuoVsqchylew1DxczzXOqQX1n5myO7u7bS0NxFGtxf7fLQLJvxi2G396AkhYH5jJVyCx1qb4bRRPayQas+uidkktSdhOqeZvMaufkC/Pw5G3kOR81WLq98UWl1LZedqUtvbXEkdvepYJLLdyeVA8SSKFVPKLSXCs48oDylBkQ5LSaLe+LptJ0+C+ScX959huvPeBESGERwm6jkGBtkLLMAuM/vl28K5jw7DWvH04lilEkF472/mJJYyypbubmFZAh+zxIYgjS7v30pIUFXwC57jQri+jur/AE3UJLu4NvcFba7nt9pni8qFyzMirHkPKyjAGQh4JVjXn95H4j/4R/xHpcNrqTi4fWR9jawxEbdzctHKkgBLytKUULuBKNkRkYkPSXWr+KRE1nYWs82r293fSuk1qUgkt9twbVfNIWNsk2oIV9/XdjD4seBdR1u++3pq7zyRx+WYWubeWOTJ3bhlrW3UrwuAqkglsnBUDl/DT+JdL0KythaX1lGsUCqYNNV5JJ1sbMRxSKQMxlvPVnYptMaqZYwBVjwnP4ll0s+IDPPI17d6ZI9utou26E1tZxzzMcZ2qC5GzaFZG3bh8o6TwXY3cHh3U7Ge81JbganfgT3FukbpundldMxhHBDCTOGXLEdBtHH+HNN1fw7pemxxQ6lZ276PpqX1xb2AeeD/AI/JHCRiMhnErxKw2MwVyzc5cXG1jxjLbaaly+qw3cktwJ4bTTvLcD7TIqfM0E8P3FXaGkjA+80jq4ZblsnivR7CWbTUkuZbjU9VSHT5bYLEil7qaKR2OGy0ixgMWVCkgG3OHNfTdZ8VS2Uy3c+pJaLcRCa6j0+WS7hjKS5KK9nErjesC4WKQqHckgbSupoza7ZfDuSRrae31NtQuZJ1FuPMjje9cySJHlwWEbM6KDICQuPMyA2fb6r4ldp11G61y10xbuRYb230hWunQRW5hBiEcnyvuuGZtgwygHyv9XWpbXHiuaPW5rSSSaKzQNpiXluIpb5zZxFVkBVAieazk4wS5K5jEZVs+51bXotLvTY32sz24eAQXV3pLx3TSHzPNjCpanYgVYiJGgcEs65yQUjvdT8SrBo0UVxrnmSxbp1js1EzFnIBMhtGh5GNquLcqMGTBY7Og1zTtRvvGWjtZXc9jCun3qTXcMCuy5ktiqqXBRWJU/eVsqrgDPzLzdpr/ja41vSXvLSSziuEs2e3FrMYisiRmbeFt3KOrtKo3Tx7diFlxkv1ng5tduPD1je+IJcX09pCXthGE8pgnzFvlB8xiSWGAq8KB8pZ8O4l8UXFnpcNxbySSWT2cOpF7ZHW5uPtVtmaIbT8ixrM+4BQvmL0dGEefa33iPUJJzP/AGldxre2ZVZdP8uAKbyLcwjlto5FIQMww82wAlnDBGaRNT8YyW0Qtpb430n2cX63WmfubGdrmBWSHCJ50Oxrjcwd8LGp8xc7m6zQri+jur/TdQku7g29wVtrue32meLyoXLMyKseQ8rKMAZCHglWNU7rQ7fTNe0a50yww1xqDm8k2GXahivJd2Wz5f72dvmGP9Ztzg4rD1GXxHp994wl0+3u4kkeaexkgtvOa4uF0+EIpBU4TcrYIBDOu0lcBZbmoXPiXT7yWzlvL6axP2ea41G009XlhDi4DpBGEfcoeO3G0rI6rKxZjwyljqvi5H+ewnu1kija28yFIy8aXE3mPJkpsme3+z4Rto8x/uqqybLksur3XhSCaa3kluZNYhdEktgWW1N+pQlCvylYNpyQGTGThgSMv4jya7Np2r2FjHfPDNpTpbQ2lkJ1upGEizLKf4NqeWU+ZCWY4EpGysfxC/iWXw1rthHaXyLPFqK2lvaaarrdl7i5DiXAGzEXlOjbkLs5P74/IfQILHRtX1FNchj826glaFZtzrteEzwkbTgHBlnHTndnnCkYeo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz4Y1DxddXunQ3EF9PC32eW+jn0tPLt7oXlp5kcbbfmjRDKVfB4DMJZCuY9Dw3Jrt74y0+61WO+Pl6VcrcmeyEUdvdPJbF4I3GN8Y2fKSGyAxEknOyG2bxXp9ld/2ak01zJqOpiKzmtPLiRC11JDIXOCS0giGdwXa6/Ln5y2bVPFSTlLK61aexQoRdXWl7JWzHebgVEIJAdLbogPI7Nk9Dp03iE6ldJftKlqsoiixErMY18797uVcEtiLIwMYxt5DPrLLc/Zss828Hr5PJG8jpjrtFWbgNFbSGAESbWKkLu5wSP1rFvZ9UWQiC4ufLyORaEno3+wfRfzroqaxII/u45wOf89adRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRX/9k=",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAKlUlEQVR4Ae1YPY9lRxHtNzu7Bha8K+9a9i4fksFGIkJkIBAWJJAhS0ROHSAIkMiJCAkRAQEEBE6Q/wBEYAkQwkZCSARYJgKBzIKXJVgMXj/qnFPV3bfefbNvBpsxrb4zc7ur6tSpr/tuz8xmW8rGfuKOnSu4XZWbRbsg0JqtC5mQA3ALp52EdlJMjCHGCjbbb8q2HFVmVl2lM23eBIozxT3YyRJUwZuNdcC+7ZIzV0nampom2YWFSp7hBYZQhc4UjqNlhz9YYBVvf0dcZ7INLMyEUFKZ0iPQABDx7kSBCMYp5RhemLYV3x4AKqTClsz1RpiM8txSIzuZ4CNKrhEBOqFotZuHDNGsUPb3Dcjl1fCl1JDCV9H9g4HhKwNRLJhBwGI/zMgVyg56w4raVca4AqzG4BGm0iqEKcUviirSqhw8kwBA3HM5Mlsj2SikMbDggJtZyMDLsAznpQdV+Ka1Y+jd+72XbY4BhlU51DtpF14pUM15oV8fDyB9wR7PtNaQrQ8cWzMEA1b9AAYGXJGxJNzJ4Gvk61QBAq0oCJZVOt0FkF5UNSuEZNDg2GyBVtIgjXwRrDJA2E0V2pXrYCB9T4U+FbildjY3vKXRpf5KsolsXoJI3Su7PWffXqWdJZHDUh8UwVYQq/7JrcPslkQjids5vHC4n3BSsPv5nq/dCo7jT4n4KdbOL81Jan8YuoNP4zAFNjoC4x58bhGSKN2c2N2cQunQBcEYyQnCFbwWju6AIDQhUAkLiF2OWijtpYXPNz4P/Exgr22cgPz4O4Qsmu5STTsI9C4J346JGbh/Mweh+EGweN2IcYcUqaIyrpF5lcMYKMisEYjFW5oKvtREZzJflFjdlpaq1oZJJF/q9AaVc/gEtaVNQ+i7EKjXQwvTEmJmtHf4mmblXgQmR38suSv0ET3iNFaNsMn9Tl5L36rL2Rm19R3acOipuO8Msa0J9VnuOC5QYZWSBcchZiZrib5tG5suVSTIJA3aqcWJdtKr85VO2XUOTq16ZRUH7l06oazU3NQUzQxF/M4gtHPTJLhg2ndJBPnY675jCV1Pl6lWtD2IL8411H38xHsSSLErQpsqroVEXh2gprmv4ApYbOKTtFDuCIehdtxOUqxSripPYoHtCOeYdcKPLKi8L3F60ejmBqQ1sKGGt7nzCyz0chXiZLwUTiU8nFxtCs/FKQNBQ/iGN1e79cHpIIDBZTrWh9juUJky2gYN3ikgpk7vKmmgktatrq4EtFcleYyELnFjPOI8tEhhjp8+I4YXXFEWoOrLwnLq5HGyoy07alJU6tYQba0QmWAJq4OxSIU0qhBKqvINwCXaGTQrCcmpKn3TMgsL1m7fAGLC/7S2/MPKgkNlQ5Ulus41IDIFwqXwkDqkxOKiLTUC8J7agtGCmTqSbvhG0e2WmYG+MzKCV+fqTTlWXMuT3+2UhejnnSDI1RLRn8dBiwdNP7Dg4qMnc4eNo3UXv8sIEhGILRj9cQgT1sjMdV1Ap60AZMf0V1h6xv96/1bzny5By6YeS3q/ub+GdQBZAGNF07kPRZ1VI5PJAQ5v1tgtDQufgCzWJZ6mzsmjWTa14IX3qYWVuvZyHIo9FHe6QEdeOz6w8PRTkFrs9crUWVatmqMrAdIXnZwH2KBzXpMdKxvUDEvggpwm4kVCWQ9PHCo5uYhr+tVAnk45jleCc1rVeMGouzxs9RcNNbjRqg3fYrZFUjBQFo+LjqPkkIAGrq4VWzXOXyMgrYbKyckNdgXv15bOJv95qOTlXO9VqY36oZ4Ao/dzk+knMTz9JRnYtso/OkzP7ta7d+rFNjBS7gsU2W1zwWFYcFalV8H81EvgliH3etIhsLEu0DtCBD4JHRh3NvGkQBv8atlS17DCJ+JzguDVxuCo2ATXUS1wRwVF9UTGcuGOh3nDsoHAEkJa0snd7tyYKo5zWnUTpnKtBapoMiHps1w1xMJ5XbuArAlndGtUhxPsP5bQq5XHNc47ezAULxYauva5XqA1pmrBZBdXdm3GsMQaFiPQUWGKbAsM1s1ZzuGUXfDtUYf5zVkjSKyJdY+6oew3LU3MuqLzqzsifYuGhZmeFNhHnnjqqA+eBy2o6KVb3cK7A4DHw7qH+wnmJqEINbgcYoWWkcBsn3+YeQHgKYS/bPbHg0PigLMPQ3weuq0xW/PCgA2oCfATRYsMjqMZ+/i72l0aERFul9Gz6VwbWjYQ4hKxcpCm3avNNz1HLdi70z0U3RYBGh92eDkuAW7nm5A2mcOPb9PwFDZyD7bG2bs2FL0aSKKoFnd6rCZnMCvYhgPIDmIZKZmjjkUkCDAYtjqHX3Womz6iPHo/0dR7ZLkkN/PK5Q9dZ1GFVGDCqtczhVbZYla0IggHFMnD0vKutYEJ3x2T+1UHbezOxCu75KY1A8kCJZHhAwSOyKfLwD3dIhDqq03IWDoccjuz4yHkbyEmzmHNRU9k7Luw6KuM5bF/cR/dpdDfXip/6MW32z4Kbs/HaoZdeZtLpfx7FSTltrzvBOu5m/hk4pOHh5xrS0kfl2w05NNf+eTzT5bvP//dd377xeOnPrPZvvjg76/e+/gfX7vxs8ceh/9fr/34oVfvvPuBF95170uND4arFxby/17AhPUreVs9C1Pb/w93jZvts58qT27/dOV75Z/PXPzW3778+fLDJz73/ltHFx6/cOVROl8sd65/4GN/vvnwpz8sste4vFyunXe98atlPLKxIr/6P91Q+urLr79gmJ8fly9+/dFytZQbj5RLr7/jQX3YL5frN4/fc+Hy0R3J5QEQlou8n+8Nx5JdnlddqbTK/Imn6MbNS09Q/OxzN8sHP/FGeebpH5TLpbxyr9h/E+7aB9yuV68/VB65d+PKtY/couy39/bCOe3xSNsoOTWtUTt/v8UB1huBvv2N3z5Vth+9dO2ofOg7Xytf/ck3y8O3f/nK3e3v/nL39RdYx+bvt37x01/94zcvP/ejvq43yp3bvXwee1XTR97V9Nb/+307lqIU/4iGONq6W/BoFaZ6ZsGpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpw4JzzcSFNBc8KpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpw4JzzcSFNBc8KpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpw4JzzcSFNBc8KpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpw4JzzcSFNBc8KpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpw4JzzcSFNBc8KpIcOJc8LDjTQVNCecGjKcOCc83EhTQXPCqSHDiXPCw400FTQnnBoynDgnPNxIU0Fzwqkhw4lzwsONNBU0J5waMpz4HzLrh7+qwGs2AAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen1.to_PIL(binary_method='gray_scale')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen2.to_file('readability/gen2', binary_method='gray_scale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APR9fg1238Q6/q2lPOn2bRLeWGKO1En2yeN7plhyQcryAyqA53phl/iy7q+8R28kFrbf2lbh728LRWGn+WzKbyUI29raSElk2kbmi673dg+5djxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdK9vqfit7+zF9Hd2+oM9oDYQWgeyeJkjNw7zbW2ujG42r5q58qP5W3fPl28ni671q2vhHfPfW2lTvIk9kkMa3XnWry2cchwDGwjKLIQ2AzMJJMEJ1EN9rOq+DdX1TTJM3V7FNNoybU+RPLxActgHeVEvzgbfN2n7tU9R1nxXOQ2kaVICzzXFqs8QRZYBbSBFm3MCjm48o7fkbY69CsoTH0jUfGWoS21oby7hikvUSW5msJDLHEYLgvzJawIBuSHaQrbXPzbgwQ6FgmrzeM9BvNQl1ICO31SyOLQeW+y4QRPIwj+QyRoHzlVJRdoAJVs/Uodfg8Y6m+mwXdul1eqv2yKzWRkRv7KjZkZ0YAbBPnt+7JIOzg1TWfGGlWIkilu5pJrhoFM9nHhDHqEFvFgYjBM8Tu53MATyhjUcdJpt7rr+GdXmtknvbqLzP7KfUYBby3WIlK+amI9v70un3Uyqqe+48/b33i6816KytL/VRpMksCJf3ukpHIYzFdmUkFVwwdIVBZEAPl5Rw2Zafie68V3nhjVrB4dSd2sru2tUg08SfbiJJ4mM3A2HyVhkUqYwzSHasnEddZrmnajfeMtHayu57GFdPvUmu4YFdlzJbFVUuCisSp+8rZVXAGfmXm7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJfQ8S6zr8Hiq9sdLluykOmQXFrDBZrKst2zzhI5XwSkT+WAzYAG3PmR/xx3OreKYtejVWvkh+13Qu0/sozW8EKxTm2ZGXDybwsbMqlzvwu6IkI1ODVvEw8NXk73Wsi5NxHHG0lkzkrhiwTbYq6HpuZ4JExhVYMxKR6nrPjW0g01rdb4IbQSiS4t2kaacu+5Zlgs3bywoiwNtu5ViD8+RHoNJrt5420cXUd80Nvqty80f2IC3toxBcJbskvG7ejKW5kAcgExHCNoX9/rqeKGhhe+GLuBLa1SzDWk9qRH50sk2w7JFzPhfMTPlx/K2758PVp/Eeu+CrrTmTUlu4vDlzHqYFjsMt9ti2pGWTEgbbcLmLK4bgglDXaW+qRT6Ldm0u55biyiCSS3Fk5kEnkrIC8KqjMxV0YooU5O0YPA4OY6/qvhG4lvLrxAHg1OwlBW0V22LcIXKRmzikYqPnP7sj5FwzfvFrcutX8UiJrOwtZ5tXt7u+ldJrUpBJb7bg2q+aQsbZJtQQr7+u7GHxn3kuqXEmlz6Rc65rF1bagZNuraZ9lWIG0ul3L+5hDcsBgsAW2KWj3bq3NctL7UI/Cf2C6uw63u+S9ltf3saGznUyOhVQjncANygK7DKn7p5/WNY8aQ6Te/ZxqS3lvb3CWHkaas32ySOadC0wK4U+UkMi4KBmkO1ZBiOrGiaDdnxjpN/cpdpFavrjqj26bEaS9+Q7mTcpdHJBDDIQY4L7rGrJq9j4z1u8spdSYT6ZYhdloJI441uJBOY2EZ3SxxsXVCWLGQ/K4CqNTTb3XX8M6vNbJPe3UXmf2U+owC3lusRKV81MR7f3pdPuplVU99x4+4u/FMGo6pfaaNVuLeSKxgF/d2BguDCpu2YhFhclhIyL/AMe+djA7cYmNi/1vxqPDjzwfbhfrp8jWS2umNILmYPKGEwlijKMsawsp2xB3ZtqSDEdSSaTq7+MrtLe71JC3iNbgXElkGjhhbTHQvG2wKTnMQJLBSse5SSd/SfbddXwn55Sc31tqHlyN5A8yW1iu9jvsA+Znt1LfIvJb5AMqKy9SvfFF5c2UUmlSRRR3ET3Ef2dJU2tfWxhIPzfOkIn3lT8jIW4HlMcvQNE1k6xawLqOq23ky60s95NYoHAe7hdBGWjEf7wDfu2uDmQLjA2dJLd+I9R0Pwo8Rk068v3jOplLXd9nVrWR3G187CJAoBbOG2ghhlW5vVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNlh9Z8cJ4j1GGG23sPtQtrWWGUwOFRzblXFuqKzFYixa5I+ZxhWIVMuWPxHL4pOpafa6lqLwJEbC41Gw+zGZltdSwJAAgAEsipkrHwycch3uW2u+Kzod/5txfCRZYRC5065ac537wG+wKqr8qEHyJMfMCQXQr3Hhu6uJ9HtkvftxvPKEsjXkQViGZscrGin7vTarhdu9VY4ri9HHivRPCehWqtdyiXR4TcKLIBtPEbWyERrgsZRDJM2195Z4vlQDKHQ0XVfFEd1c3d/BqV7pkVxLbwxGySG4lVorZoJCvHBbzlY5XDSZZY1VhFTTUfHFl4j06yv33qv2WKWSK3leC4BRBNI3l2rbW3mULmWIDahZNuS+x4Un8S/8SU61PPctf6Ubi7ElosK2s6+SFQAAFWYSSFgxOWQlQgBWs+3fVNF8T65cmTVZrFtbjmul+weYDbNY7VMWyPdJiYRodm4qI1LYy7Nc0O48V6hrkyXckltpexblJJLcLIyG6uSkagrwWhFvv3fMgGNody65a3/AI5tfDlhOXnvb+90pbi5V7MRCzkV7cMFCoxEnlyTMVZZCXi+SPGYyWWseLJdOgM73wUSzLHLFpzyTSygRGGKVZIIR5bFpsuqRKAiAyqdxbQszqmmfDfwrAi31nstLaLUJLa1825tkFufuxFHJbzBGhGxsBmOBjcsfha78aalqDR6uZLO3RBLK0lqsblhd3Q8tB8y4aNIdx3vhQNpJk8wV7bVfFZijlzqUsqJDNqsU1gEW0cTw+bFbfuwZkMRuRlTKTsTa2WG+5Fc+Jda8RlLe8vtO0Sf7RJFMdPVJQiJaBABKmY2MhuceYp3LuwMbGXHur7xTrEGlfbrO+imlu9KnksodPJgSHfbySys5+ZJFm8xCu5iEXJjxmUbHjjUfEtl9t/sX7csiaeZLBLSxW5W4n/eeYJcj5diiNk+ZdzMQBKR5dRvd+NLe3e9tzJeXM17qFtBYS2qxxRxILlrd3bhss0cKhyyoUkHG47zY8Ii+l8Va1dXDalPaNZWcVrd6hZfZpJgr3JYFcLyrPj7iHG35SCHfLsdW8Uzrqk3narKqbNr/YTEkcbSqJHjilto5PMWLeyIGuB8uG3Hbvk07VfGhurvEF3cws9xb2DXtksAZjFatbyTBcFU/wCPlnPy87l2q+yIaGqvq+r+B/GYmsrtfPt7iPT7V4h5u37KqlNq53HzvNAPO7gqSpUmn/afiXypPMl1VLHzYhd3a6YoubZysplSCII++NXW2UMFm4lkPmPt3JntrHjGW201Ll9Vhu5JbgTw2mneW4H2mRU+ZoJ4fuKu0NJGB95pHVwyyXMOv2HiHW5NMgu767bWGubdLuzXyMHS3EZWXYoA81ViJD5AVQSC7F5Le+8XXmvRWVpf6qNJklgRL+90lI5DGYrsykgquGDpCoLIgB8vKOGzL0H23XV8J+eUnN9bah5cjeQPMltYrvY77APmZ7dS3yLyW+QDKis++1Xxc7/JYT2ixxSNc+XCkhSN7iHy3TBffMlv9oyi7h5ifdZWj38ver4p1TVI55LK+vPst2EsnurM22+EXOlyIZNinbnbKzHbkbX+RNpRdxZPFFz4gs0t5Lu2F0lvHe3kmlorR4F+ZUBwQAjCEISzqNyHMgcmSnr+v+LtO0HUrnzL63msrS6+ySJp6SLctFLOjSTkgKmIUhlUgorM7bVkGI66jxXbXEuteE7mKa+SG31UmZbWASgBoJVDP8jFVyQhbgASEkghWXDttV8VmKOXOpSyokM2qxTWARbRxPD5sVt+7BmQxG5GVMpOxNrZYb7l9qvi53+SwntFjika58uFJCkb3EPlumC++ZLf7RlF3DzE+6ytHvp6Re+NNQ1t7NpruLT3e5Vby7sFhdYQloYpAu0gytvnwG2jJcsg8vyRI0mu3njbRxdR3zQ2+q3LzR/YgLe2jEFwluyS8bt6MpbmQByATEcI1jyNdsde13UbV5xbvrdoqWi2oK3EbxWsUsjMQWKou4gptCtG24sPlGHYa74+cSrLbyG8L25ktpIJQkbm5hWREk+yqgi2NKGbfOQAGViFLHuNTOr6f4dR4rmS7vYriCS4kitxlofPVpgkYBOBFvCqNz4AGWfk8/qV74ovLmyik0qSKKO4ie4j+zpKm1r62MJB+b50hE+8qfkZC3A8pjl6BomsnWLWBdR1W28mXWlnvJrFA4D3cLoIy0Yj/eAb921wcyBcYGzcvU1rXNK8E3RSSw1CS4Se8ZLYn7IWspw/ytnaQzbRvyAxXIb7px7jUfE0A1KVku4Z1SG2juU0xmluFjubxc744JcExrE5JjZBvO1U80MNjUYtX1D4f2AH2t9Ta9sZN13AHdcXkTb2SNY8oFG7BWNto+YI27Fe6vfFFpdS2XnalLb21xJHb3qWCSy3cnlQPEkihVTyi0lwrOPKA8pQZEOS1fQr3xpLpsOo6hNd+ZHcadEbFrBU8xZYrYXDv8u7CNJMw27drI24so2rni88TNqUWoXH9pLcQ6PcJLI+lMVtrt5bXfBCY0Yug2/I/lyActumCsq9Jpc/iK58A6nNG98+ttFP9ja8SNSZNmI9oMMPy7v+ekanO7qu1jl674l1qTVrOTS31K20m5uIbYltMInyIbuScpFKquxCrAwIDDK4VZDuRus8OXt5daYiagk4ul3MHlgMZkhMjrC7cACRkVWZAAVLcqmQK5PXNEufDV7bS+EtNkUx6ZPGzAu4QB7CLIJD/OIISVG1yTHnY5yDXtZNe1I+GbzUbjUh9k11lJisXLmJrZwGm32sZxvYpvVFQLJydw3jc1+TxF/b8/8AZ91fRWccVhGkcNtG6M01xJHO+5kYlo49jjB2rgFgVJB5/Vdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNnQeOr/AMQWX2AaOs6Wr+Z9omtkZ5FcbfLXC29wdpBkJPl9VX51+6xaaj4gmnsjP54tZfsrXksViy+VKyAtHEsgDmEts3MyFo97DP3jb8vpeo+JfEHh/RZbv7deef8A2bdXnmWKxJC4u7V0aEoMOrR+bI5DNt2jIi+5Vgah4uur3TobiC+nhb7PLfRz6Wnl290Ly08yONtvzRohlKvg8BmEshXMexoOqa7p8Gh3HiS7nlXUdPRrrzLIRLa3TPbxxRgKuVZzM24OT8yEqEUFRc13w5aaZ8O/EWnaJYyAz6ZKiQxl5GdlthCgAJJJ2xouB1x6k5y9evPGOl/abW1lnvY/9GmlvUttjRB/PEiQhIpcqrRwAKY5XCync2MOupDqWst8PDezySJqmxlDpZTs/wDrCqnyzCH3lcZbydoJL+WUG049tqXij+y7KR5NZ/ePOfNNkkzyTDyxDEU8mErA2ZdzSJEQV/1iqVZqek6h4uFlHNdwX1tNP5cskkGloZJ7r7HZ7Y5F2j92zm4DOSm0xqvmxgAV1Hhawivvhvpui6laTqo0qCyvIJo3iPNugdcnB6NglehyMgggV9X/ALR0K6XT9G/dw6xvitW+U/ZLx5Xlllw2d+UeWXacKPs20cyAVn39/wCLnn1GHTXvjqZ+2Itq9mi2kEYSX7NLHMyBXkZhb5UyOB5kmVXadnSaPqdzqus308cd2mkrbwJALq0e3bz90plwsiq5G0w8429cchq830ybxfo/hmwt7axu7OAJAk0kVoyzhlsbQKpC207MN3ngsY+DGF3rgK2xqN14m1I6fHdw3Zdr3S5Xt7XT2+zGASW8kkxdwHRxKJF2MSwRQTEOZBseJ4tXg8Y6fqOlfa5p00fUI7e3EAa3a4/dNGkj7coHK9S6gmJQCMsG0PC91fXFxerJc6leaeiRGC51Oy+yzmUl/MTZ5ceUCiIhtnJdhuOMLz+kXXjCO106e7udSuJPs+mTzxT2Uah3uZfLuY2CxqQIVXeACCpYlyy4APC3iPxDHqUn/CUmS3idESSN7eYrFcvLHGkcTi2RNm6Qr/rJiflIYgMx6y30my8N6Ldx6RD9mVYgw/dyXODHCsSHYDufCRINqkFsepzXH6dqfiV9JeW6uNcWE3aR3FyLNZJYIvLc5hjNpE7MZPKVswuoVsqchylew1DxczzXOqQX1n5myO7u7bS0NxFGtxf7fLQLJvxi2G396AkhYH5jJVyCx1qb4bRRPayQas+uidkktSdhOqeZvMaufkC/Pw5G3kOR81WLq98UWl1LZedqUtvbXEkdvepYJLLdyeVA8SSKFVPKLSXCs48oDylBkQ5LSaLe+LptJ0+C+ScX959huvPeBESGERwm6jkGBtkLLMAuM/vl28K5jw7DWvH04lilEkF472/mJJYyypbubmFZAh+zxIYgjS7v30pIUFXwC57jQri+jur/AE3UJLu4NvcFba7nt9pni8qFyzMirHkPKyjAGQh4JVjXn95H4j/4R/xHpcNrqTi4fWR9jawxEbdzctHKkgBLytKUULuBKNkRkYkPSXWr+KRE1nYWs82r293fSuk1qUgkt9twbVfNIWNsk2oIV9/XdjD4seBdR1u++3pq7zyRx+WYWubeWOTJ3bhlrW3UrwuAqkglsnBUDl/DT+JdL0KythaX1lGsUCqYNNV5JJ1sbMRxSKQMxlvPVnYptMaqZYwBVjwnP4ll0s+IDPPI17d6ZI9utou26E1tZxzzMcZ2qC5GzaFZG3bh8o6TwXY3cHh3U7Ge81JbganfgT3FukbpundldMxhHBDCTOGXLEdBtHH+HNN1fw7pemxxQ6lZ276PpqX1xb2AeeD/AI/JHCRiMhnErxKw2MwVyzc5cXG1jxjLbaaly+qw3cktwJ4bTTvLcD7TIqfM0E8P3FXaGkjA+80jq4ZblsnivR7CWbTUkuZbjU9VSHT5bYLEil7qaKR2OGy0ixgMWVCkgG3OHNfTdZ8VS2Uy3c+pJaLcRCa6j0+WS7hjKS5KK9nErjesC4WKQqHckgbSupoza7ZfDuSRrae31NtQuZJ1FuPMjje9cySJHlwWEbM6KDICQuPMyA2fb6r4ldp11G61y10xbuRYb230hWunQRW5hBiEcnyvuuGZtgwygHyv9XWpbXHiuaPW5rSSSaKzQNpiXluIpb5zZxFVkBVAieazk4wS5K5jEZVs+51bXotLvTY32sz24eAQXV3pLx3TSHzPNjCpanYgVYiJGgcEs65yQUjvdT8SrBo0UVxrnmSxbp1js1EzFnIBMhtGh5GNquLcqMGTBY7Og1zTtRvvGWjtZXc9jCun3qTXcMCuy5ktiqqXBRWJU/eVsqrgDPzLzdpr/ja41vSXvLSSziuEs2e3FrMYisiRmbeFt3KOrtKo3Tx7diFlxkv1ng5tduPD1je+IJcX09pCXthGE8pgnzFvlB8xiSWGAq8KB8pZ8O4l8UXFnpcNxbySSWT2cOpF7ZHW5uPtVtmaIbT8ixrM+4BQvmL0dGEefa33iPUJJzP/AGldxre2ZVZdP8uAKbyLcwjlto5FIQMww82wAlnDBGaRNT8YyW0Qtpb430n2cX63WmfubGdrmBWSHCJ50Oxrjcwd8LGp8xc7m6zQri+jur/TdQku7g29wVtrue32meLyoXLMyKseQ8rKMAZCHglWNU7rQ7fTNe0a50yww1xqDm8k2GXahivJd2Wz5f72dvmGP9Ztzg4rD1GXxHp994wl0+3u4kkeaexkgtvOa4uF0+EIpBU4TcrYIBDOu0lcBZbmoXPiXT7yWzlvL6axP2ea41G009XlhDi4DpBGEfcoeO3G0rI6rKxZjwyljqvi5H+ewnu1kija28yFIy8aXE3mPJkpsme3+z4Rto8x/uqqybLksur3XhSCaa3kluZNYhdEktgWW1N+pQlCvylYNpyQGTGThgSMv4jya7Np2r2FjHfPDNpTpbQ2lkJ1upGEizLKf4NqeWU+ZCWY4EpGysfxC/iWXw1rthHaXyLPFqK2lvaaarrdl7i5DiXAGzEXlOjbkLs5P74/IfQILHRtX1FNchj826glaFZtzrteEzwkbTgHBlnHTndnnCkYeo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz4Y1DxddXunQ3EF9PC32eW+jn0tPLt7oXlp5kcbbfmjRDKVfB4DMJZCuY9Dw3Jrt74y0+61WO+Pl6VcrcmeyEUdvdPJbF4I3GN8Y2fKSGyAxEknOyO2TxXo9hLNpqSXMtxqeqpDp8tsFiRS91NFI7HDZaRYwGLKhSQDbnDkg1vX7O41DUJrrUp9B05La4MuoacttJPATOtwx+VTmP5JMBEYiFVCnfvevrWq+OrE2TBJIzJb/aNqQPMqzNI7G3l8i2mLJGvlJuBhL/ADEMSfk3NMn8SrqNpcXk880Fzqt5aPam0VEgtVNw0UpIG7cfLiUMSFKuvyljvNfW9R8Sw+IfKs/twj/tC0jSGGxWWBrJni82ZpCPlk3GVCAxIQbvLA/einbXvjS30TS5rma7ubi+0xJ74iwVWsWD26v5aBc+b5Uk77G37ni+VAMxnU8HQajNb+JGu3voJLrUM291NarDM8f2WBEl2EFd3y85UDcCCiHKLyfh6z8R6f4f+3WT6lFPY+F7Pbbz6ftkubmE3R+zEMgOwEhSFUOwMZDjOX9cooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooor/9k=",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAJlUlEQVR4Ae1Yi3bcuA7L9PT/fzmXAAg9PJ50wu62l3vkZiyJBECK9FhJH58fH4/4+I5ZGji9XU+PZhbQePVua0LewG2kp4SeUrwoeukRajF/fHx+/BjK3PVYlSb/gEQp7tukSFAbfjyiAvETl8gctdI0zHTJLyxMYpoFBZtsC0Pi6HnStwq80l3viJtKMYGHmRBKqTBmBDoAIj5JXBDBOB8fP8FCt2Pz8wGgQSZMqTxuhMkp5ict8lMJHElydATYhKI3bhnSy/DCuN4fEBdr4j8+RkjhxzL5VmD4oUAUN8wgUIkPM0qDsoM9sJJOUyjeAIfTOsIMWYUIo/QlMZb0KofMxAAsX1yJvHqdrDcyFbhhw8MtpPFy7OFy65Yy9zIuCit9nee2g2gwvMph3Cm7sS6BRs6b/b49gKwbznhhjYJ8ZsMxDYcVMOoDGBRwOWOtcKdCjs43pQyCrCQIllc23QWQXVIjK4RkUGs8PoFW0hB1vgg2FLB4ThXWm+ttILnfQn8LPFOr0fCWRpXW67KOJYt3gci8Gpc5ez9fpYvnIg7PeFAEu0Hc8i+0BfO8JTopPM/hjfCrxVfBfsX9u/7YsI8/JZKn2Dy/1CeZ82FYDj61IwyY6Aj03XrpEZIo3VI4aSmhdEhBMEZKAVOhG+FIBwShCYFJWEDiStRmjJcWvt/4PvA7gbmmPgH59U8IVdTd3Uw/BPQuMXdRYgbJn24LSh8C2+tGik+iSBU74+jMx9pOo7DmHoHY3tI08KUmuVjzRYkxfZdhmDVhEhcubXqDimyOpSNtOmxfQmC/GVqYmRAzo3/BjzSH9haYGuuxlFTYHd1xpqpaONfrTKydO2zX7EI66g6rCasU54vD05HQmuUTcUPZKyM37EMsXFES/cTUkyVVJMgkA7qYpYlykrVwZVN2CyGltV95pYH7ko6NQ5qTkWK4YfDvDEKnNl2CC6b5koTF/9vjq2MJVb9cYbqxriC+OO9Qv+BJ9yuQYg+EJmN5FxJ5LYCR5qsND8A28TdpMz4t3kM90b4y3EreGr9Sge8HzrGoRB5ZMGVdfHrRme4JpNdYm8EOOv9Bhaw0Ic4VL0NKCQ9SmsOQuaSkEXSYazbHuK3BSRAg4HL91Jc47jCF0WWDBe8UCNOmd5UsMMma3jQPAfqHkTohQopvjEdchpYo3P6sGTG84IqygQaXG7umTp0U+/HJisbKO02vlzEOiFzw2JtgDDIhjbGwkabrDcAdnQrqlRYX0jDmZGZmD8ZlPgFSwv9pffIPqwgOUzRVHledoyFyGZErM2T26qKSyxhGBOAztU0xgoXZSU/8lFhme2aQX5yMkLtL8+Pjp+JGnvyZpyyWed4JglwjEf15bFk8aPrAg4uPntwL1kfrM/5ZESISkJoV83GwC6MzS9sSMGUHANkx/RuVVfG35/+2/vcSjGzGsaT3W/LVrDfEDPSIonNuw+jVFJMrAQmfXs92x8YxZBt3PF0LKaNFNmPDG/vbi5t9vdR4F/su7nuBfuTe8YUFM09BWjHXK1Nn2fCqj2kESP9ISh1gLZe6sU6sfDAzLIGbOF3ES4RrPTw+VK7JOW7YbwNlOh8//UpIzdg1XjCqLg9b/UVDC270asK3WEyRFBxcSyeXieMqIYYaN8aBHZbUHxGQ1kRdkxMNfgVfx5nO4/rnoZIXedyHURPVQzUBRu/nuSZPSzPzJWnsHMV3hclcbit9MW9TY2R8FcjZfV43bMemOYy5C+anWgK3h3zJJMFYjxv6aeHAX6GNSXIsvwr0wK+WM3U1yxzHZwehq0nAseNYpI1mgRcpGAYTGYvCGQ/ziWUBgSWEspQTPe6chMnHOb26CTO07gINNJWQdOUaITbyvXWD3C2KtCn1vsDrYwm1unlcfd7Fg6F4HuhYypd2ge6Uhged3a4rdTrt8WhPCOioCMPVZwzGR+UcvmRnvRdmu/+Z0UE8XlRfmCcqftNSx6IqOr+WIzKnKJjdZHLBOvLEU0Wz8TxoIUWWbmMK9gKAToZNRvIES5dQhAZcBI+wMhKU4/sPNy8AMgXz5Ys/HhLiAy6+DP4+LNNQjuLZgQmkCcgTRYMciaMbc/9dnZQpRET65cxsFupEywdBXBJWDrLM+/DlZNUYG87qLA/FMkWAqYcZXo47IP18E9Int3l8m5oprHO32tRcqRNF1gRpKantTsZtcgGLDUdzAHlC7JEubu9ji4QFHIEdZPMGYUzWiGKsPMmMu7PcxcN9c+VDt3i0QxrQYe03M4VV2aJX9CIIG+Tk4Zl5j71BCT+LUvIGQZO4M/GhrvW0hoNiRmnJ8AZBw/ksGSQzPQJhf6MIVywJ79zKxHfE/0WMz2H1RU+k50tY1FVOf5ld3QX0Yjq4mrxA/SmzNzyfj9vI72/vlv5/ZOSGdWRFB9UDnGi4dCyGkS1PJ+266UkIDo/dWGULk8HlsPHJ8Arqf+nCSyu+jkzVY6aib6mNHgXHOyC3R0Oeh6SkC8OE8aVB01/aqMPmI+1H1iPc4/90bcxxWcZbEHvUL1a2g5ofyuCma0HY9MfH/MXD3fLIPLAV5Gijx0xyLp82Ml2T/ce3dh9Q53AelzpFnT6y5gGsY3M7e0HgIYeKyD/008X1MmftBuivTZ7TeLZ8ldz30F8p/SFfu4R/ty7zHP5dpSb8s+EmjSqneTpcLl0T4ulwk0aV0zwdLpeuCfF0uEmjymmeDpdL14R4OtykUeU0T4fLpWtCPB1u0qhymqfD5dI1IZ4ON2lUOc3T4XLpmhBPh5s0qpzm6XC5dE2Ip8NNGlVO83S4XLomxNPhJo0qp3k6XC5dE+LpcJNGldM8HS6XrgnxdLhJo8ppng6XS9eEeDrcpFHlNE+Hy6VrQjwdbtKocpqnw+XSNSGeDjdpVDnN0+Fy6ZoQT4ebNKqc5ulwuXRNiKfDTRpVTvN0uFy6JsTT4SaNKqd5OlwuXRPi6XCTRpXTPB0ul64J8XS4SaPKaZ4Ol0vXhHg63KRR5TRPh8ula0I8HW7SqHKap8Pl0jUhng43aVQ5zdPhcumaEE+HmzSqnObpcLl0TYinw00aVU7zdLhcuibE0+EmjSqneTpcLl0T4ulwk0aV0zwdLpeuCfF0uEmjymmeDpdL14R4OtykUeU0T4fLpWtCPB1u0qhymqfD5dI1IZ4ON2lUOc3T4XLpmhBPh5s0qpzm6XC5dE2Ip8NNGlVO83S4XLomxNPhJo0qp3k6XC5dE+LpcJNGldM8HS6XrgnxdLhJo8ppng6XS9eEeDrcpFHlNP8HFkVFgazTfJUAAAAASUVORK5CYII=",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen2.to_PIL(binary_method='gray_scale')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf6w/EEV3PeaHHa+Yga9kEtxHAkjW6m1uAHBZWCHcUGcc52nIYg4fh86pp/wo0H7Ot99stYrMXEc9rtmSJJEE6CMoCdsYkVcKWYAEFmIJL7VfFzv8lhPaLHFI1z5cKSFI3uIfLdMF98yW/2jKLuHmJ91laPfj3GteLFsLgfaNVXGoMlq50p45pbYQQlXYpbTquWZyQ0aksTjytjRV0HibUtW07S/DiWs19FNdXawXjfZ4p7ny/s0zudi/IZAU3fIGGV+VXHyNz+qa344t1s3hjnSHynaCU2krG4IlkCG4jitpWXMQgYqPIOZHAwRiPQ1CDxTqmg3Dzvskj1uNLOH7KXJjTUkKTSqApKrGi4CkAoGdmJYbJLq98UWl1LZedqUtvbXEkdvepYJLLdyeVA8SSKFVPKLSXCs48oDylBkQ5LZfhzX/F17cxR3kl811FLYRtbnT0WJt9tbPdec4GY2QSO6qfL+Y7QZPuLueEL/Wr/AFm+Ooz6lLbqhMZltDbwZLcYSS3ikU46KHmAGQzZ2ls/VtV8VxDVTYnUn1BEvcWgsAbeGJY5TbyQyeX+8lZlt8pvfmRxsGMJJr154x0v7Ta2ss97H/o00t6ltsaIP54kSEJFLlVaOABTHK4WU7mxh11INQ16TwDFdsJDqRcLK8ds4lSHztryCOSNGaVYctjywGcfKhBCnHew1PXbzw5JdTalPY2eumWGW6sY45JYltXYSzI0Q2ETB41O1OCpGWKPR8RdOvrm/eS2OpSpL4c1W1WC2tvMjaVkjKoxCEqXwSMsMmJQOrBpNevPGOl/abW1lnvY/wDRppb1LbY0QfzxIkISKXKq0cACmOVwsp3NjDroLeeILvwFZyiWeLVZbuCF5oLZhIkRulRnKTRL8wiyWYxqucsFC4rPN/4uto2G++nWX7SkkjWaF7eOC7jhEsYVBuke3aWbaQ4ZlGxdvyGSwuvFd7qlvZ29zdjS5UuQNRvbIRT7B9m2uUMagS7muERWVAVAcrJt+c0zVfFT+PHtL5JI7FridDEYJWiEIDGJkdbbYHYCMndcMPmcbQxCpoXF5fDxY8F3fazZwC4iWzhsdO863uIiqEtLL5L7CZDIp+dMKqnAzuOGmp+MZLaIW0t8b6T7OL9brTP3NjO1zArJDhE86HY1xuYO+FjU+YudzSa1qPibT4ZrKK81KQ296Y4rlbBjLcxeTE+5nitZowA8jpgRLu2j5gUfefavElreXYWHUoInuAySpp8Ms884tbTy0m2AIyMxnDyBkUGMKJYwBWX4evvGGnN4b06S3ntrOK0sIBC9tMyNGYoxKZNls+2QMZVG6aMDYpZduS+5bXXiseENIvtVubuCe6SAX4srIST2ieSxLiMxktK8vlh12EIrEBRtMhy4JPF0t/per3Ed82rrp+sQxWj2SJAZhOpgSQru8tXVFwTLg+UuHOWL9B4F1HW777emrvPJHH5Zha5t5Y5MnduGWtbdSvC4CqSCWycFQOf0DRNZOsWsC6jqtt5MutLPeTWKBwHu4XQRloxH+8A37trg5kC4wNljQpvF2saqguDPokN5Ebm7eHTkVhJ9lsMKrSKcMHadcuHO1HXqoKRyeI/FezS7q7W7sZ7i40xJbVbAfZxBM0AlZnflJfNeSLbuLBQD5Y/1o2PCF/rV/rN8dRn1KW3VCYzLaG3gyW4wklvFIpx0UPMAMhmztLRvqOui81HZcaqdXj+1fY9MOnj+z5Nof7PuuPK/iURsf3w+Zivy/dFeLUfFMl9aQ6RcX15YyXaI15rGnmB1zDcGQMgijPloVt2U7V3MxTzOTtuF/FFtf3s3267uba01OztbeBrNP9IhkS3WaV2VQWCl5XBTaFZW3blAVebtb7xhqpW11i3nLC7sZ54Ps0xEMiXtuWET/Zo08tV8wn97McKpDEBmaxp2oeLrfTtDhEF9bzD+zIntIdLQQC1YQefIx2jypA5lQoD8qLnygP3q7Gvwa7b+Idf1bSnnT7NolvLDFHaiT7ZPG90yw5IOV5AZVAc70wy/xU9Q1XxpHqxTToLuef7ReK9pJZKLVFWGc2m2b5SRIViL/M2GO0tF9xtDwiL6XxVrV1cNqU9o1lZxWt3qFl9mkmCvclgVwvKs+PuIcbflIId8fRNV8dXc08d0kiXCJHPNbPA4AZJozJDA7W0cWGj81BmWXJ2EOAGc2PEGq+L0C3Vkl3Dpt1cSPEY4G+0QxiOJY1ZFtp2AdhcSfNHkAoGZD+7Ox4YuPE2pXlxNq0kdvaQvCyJHbtH57Nawl1AkUMsSyNIefnLfKSoQq+X4ok1261pbRY757ddV09o4IbIPA1qs0DtO0vBWQSB1I3H5Fz5eCZRXTU/GMltELaW+N9J9nF+t1pn7mxna5gVkhwiedDsa43MHfCxqfMXO5jW9R8Y6ez2du99LaW928X9oi3/fzDyoHTIjtZlK7pJ1JWFR+6UFg2Q+p4p06+8S+AdNguTILi4uNPe8jtbbKsDNF5mY5kYhFyXwygjYN3G4HP1bVfFcQ1U2J1J9QRL3FoLAG3hiWOU28kMnl/vJWZbfKb35kcbBjCbniKz1Sf4eeI4rk/ab+60+52W9sm5Yy0RCxR8Bn+pGWYkgKCEXn/E03i6wklhtTPqM0ETXcF2unI3lztaXwKRAKcKrxwBQSz/vdrOwcA6B1LWdB1Ca41K+vrjSLO7eCeae1TH2VbM3BuSY41O4Sfusr8uBt2l/mqTxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdI7XV/FJiWzv7WeHV7i7sZUSG1LwR2+23N0vmgNGuCLoAM+/ptzlM3PBaTaV4d1Nb6XUpnt9Tv3Y3FoQ7KZ3cFFSNfMDKQ+VDZLkDoFGfYeIJk8RX+pxeZdeG57iVUuNOsjciaQQWYjbMKM7gFblS/IBXaTkKBXsr/xWdb0Wz1CfUjILe2F9HbWgRBLsUy7na3aJxuySyTx8fKql1G/Ls4fEumiGDTmvpLpdQv43ur3S1byxJqdtzkImY3iaSQ7SAfmII8sbOk1WLV7jwP4z0uf7XfSw29xb2czwAS3Staq4OEVVY75HQbVH3QOSCTHqFz4l0+8ls5by+msT9nmuNRtNPV5YQ4uA6QRhH3KHjtxtKyOqysWY8MvN6ze+NNR8GanbX1vqQu57K8jhtrfTFYXLGW4R0kALeWFhEJjIkG4scGc/Ke81mPWbrxDptlYX89hYSWlzJdTw26SMHV4fLCs6lVY7pOobKh+M4ZefsNZ8U3c+nTXcV9bX032Nhp6WBFo0LpEbh5JGQmORC1wAhkU/uoxtbd8549tNROsLd2Qvp8+H9Ugjt4rJbiF5isbIjgxt9/B4Y4YxKo6sGjubrxhDFfXUVzqUjqmp3cNubKMrut59ttAMR7ikqNk8722Aoy85saZqvip/Hj2l8kkdi1xOhiMErRCEBjEyOttsDsBGTuuGHzONoYhUNWTV7Hxnrd5ZS6kwn0yxC7LQSRxxrcSCcxsIzuljjYuqEsWMh+VwFUamm3uuv4Z1ea2Se9uovM/sp9RgFvLdYiUr5qYj2/vS6fdTKqp77jh6Dq/iqe3nEjXc86XEP2ZLi0lXzdwfzFlke1t1SIKoYMqllYc79yxNXutd8UDUprW3n1Jp49MiuI4BpifPfNLcKIZMbjHAxj27sldqKfPGd0ly/v/ABc8+ow6a98dTP2xFtXs0W0gjCS/ZpY5mQK8jMLfKmRwPMkyq7TssXep+INY161GjS31lpE0sMEk0+mMjxkRXbTELKgYdLYK5BTdgYb51OH4nuvFd54Y1aweHUndrK7trVINPEn24iSeJjNwNh8lYZFKmMM0h2rJxHXYeKLq+t7iyWO51Kz090lM9zpll9qnEoKeWmzy5MIVMpLbOCijcM4bHsbvxowkv9QMkUkV7YwnTYbVTGVljthcNv8AmZkRpJipVhtKNuZ1G1ZNfg1238Q6/q2lPOn2bRLeWGKO1En2yeN7plhyQcryAyqA53phl/ix9S1vxxDqetxwxzoyRXv2aFLSWVMLHIbdoyLbZ5jFYiQ07g7mXYGIVeo8RWeqT/DzxHFcn7Tf3Wn3Oy3tk3LGWiIWKPgM/wBSMsxJAUEIuHqd14rbULSS2hkiv98kMrHTxNFbBruyXaj4DPEYTI5O4FsOSIymyKxdXvii0upbLztSlt7a4kjt71LBJZbuTyoHiSRQqp5RaS4VnHlAeUoMiHJbP8KDxXY+HLBpWu44rJ9Ms4tOayC7ongtVnZyRvIQvKRjbtZG3FgNqx6ZrvjMwal/aRnt5xaF2QWFxN5M+9BshKWgXb8zjO65xhW+dVffJPq3iaPw1ZyLdayLqW4kLFrJmdkAUDayWLFEBJ+WS3RmOSrbVG/uI4IdY0GbTdRiu5Ekt/s12l0oR33xjcC0eEJw2C0Z2g5APHGXarFoPibVAtlOlhJaRTI8Nm8xeZpbyaYblUseuQucAuAoBcA6g0rTL7VI9YNtJ9shcIkj+YhBj86MfKcAjE02DjDBwRkbTXN+Rrtjr2u6javOLd9btFS0W1BW4jeK1ilkZiCxVF3EFNoVo23Fh8oj8Far4qvNZmh19JIwbdnmieCULDMGUbYn+zRpsGXH+tmJwpViAzNh+MtR8SzW99pn+nN9u/tG1ksxYqYtgtblrZYZMbnaRUR2CmTDDBMX3Gua/rHjSCW6/s4akZw98Hhj01XhiVIJzaGOTbly5WJmH7z58KTHny31NQufEun3ktnLeX01ifs81xqNpp6vLCHFwHSCMI+5Q8duNpWR1WVizHhly9a1nxVamybSZ9SuLP7PviubzT5Vknm8x9yzxRWbsEUCPGBCWDHDMeVuNJrt5420cXUd80Nvqty80f2IC3toxBcJbskvG7ejKW5kAcgExHCMLJrtn421gWsd8sNxqts8Mf2IG3uYzBbpcM8vO3YisV5jBcEAynKLTi1jxoZowRqRRrIvf7tNUC1uPOtxMsDbcyiOJpmj+Vw5HDT/AHVkml1RtW0+90+51y+hjtL2K5vb3TPKntkaS0J8uLyYxI2FYqCpzhyBIU8tuk8UXer21xZLp5u0gZJSXtLUXDPcAp5MTqeBEwMm5iUA2r+8jzzj+D01d/Eq3+sS6l9ovPDmntKk1oEiEymXzVLCMbXVmzsLZ/etwQBsr3N14whivrqK51KR1TU7uG3NlGV3W8+22gGI9xSVGyed7bAUZecmoar40j1Ypp0F3PP9ovFe0kslFqirDObTbN8pIkKxF/mbDHaWi+41zwdr2pvPJba7POfPlSKyNzbTJI8hSV3Xc1rbrt2RZGFJB3Zb5lFGt6j4lh8Q+VZ/bhH/AGhaRpDDYrLA1kzxebM0hHyybjKhAYkIN3lgfvRTtr3xpb6Jpc1zNd3NxfaYk98RYKrWLB7dX8tAufN8qSd9jb9zxfKgGYzc8OtrsOj+LryK2nn1KS7eXT/t1uLc3W20gWJmXIA3FQG+5zuysZBRcuFtf1XxTYwtPrMulwXEb2t/c6atvKhe1v0kkIKAZDGNRvjUfc+U7g8mf4es/Een+H/t1k+pRT2Phez228+n7ZLm5hN0fsxDIDsBIUhVDsDGQ4zl9y/v/Fzz6jDpr3x1M/bEW1ezRbSCMJL9mljmZAryMwt8qZHA8yTKrtOyxqGt6xeXktxYf2ra6Efs8Us6aY4uYWxcGRo4ZIi75b7KpPlsuGbGCGZef0q68XXkeua/5s6X1tp74hjsU/0qe3u74Jb8FxtACqyoS53JiQdW6zxxf6rZW9mNKnu43d2Li1tJJGIAGMutvcBRz90oC2QQw2kNl2EHiKxvtT1FHvgr6rYqbSa1jL3CvDaRTSSMgIOwbjmLaqtG+Sy/KI/C0/iHXNSksvFMEnlKiXMkDxTKsNzFLG6CKQW8aFAwPHmzFtq4YgMWk8Nya7e+MtPutVjvj5elXK3JnshFHb3TyWxeCNxjfGNnykhsgMRJJzsjjTxHYeK9Wi0yW7kN5rqM7Xlp/o627WBw4dYxuCyIq4DAkxIpZS5ZrGqXviiz1mK1tptSmCXtlCNtgkkU1mWhE07yBQFl3GVSFPCDIiUfvVr29/4vtPD9jPfXd3LdXuj/AGi4P9mMfslwpgVY1SONmV2Ekm4ssgDJuCBVaMxw6fq2vJ4TuNSl1WT7Prbzyi5t4iu1beUo+DbRsqh1CgtGjbnbGf3T0Jq3jH7NE1st9cXzfZzf291Z+TDazm5gVooXEXzwlGuA0g87asatu7toaiviqDVJI4tUvpI44rFMwWUQjkee5kjncZRiPKjKMo3HaFUvvBbdj6rrfji2eERRzpHD50cUxtJW+0vHcSoDMkVtKdpjSF/l8nd5jbTjGzQaTXbzxto4uo75obfVbl5o/sQFvbRiC4S3ZJeN29GUtzIA5AJiOEax5XiK58bahZ29xPpmmSyzSPdQ2MZZ9sFiI9sjKVLbjMNzBsqrrj5VKZdvP4jOpXOsOmpQ3l/oWmPNCljmO3/ev9pEeUJ82JXZ1jYsxLkbXACjrPD8mo32k3yXF1fCMytHZXtzbLDcvGY1+do2RQrLIZFGYwCEUkMDluL0i58R6d4O0S1t21mKeDR4RZwHTc/aLwb1a3ucxZiiUrCoYmPKsx3tjcu5dav4pETWdhazzavb3d9K6TWpSCS323BtV80hY2yTaghX39d2MPjDWfX9Z8XWdhDqGpNaWlxbzwXl5pixTxpJb36SSMm1cHcoRWdFAIT5HHMnQapq+v2XgNrqPzBq0eppZxNJbrumT7cIVYoSikyR4PBQHflSgwRXsLrxXe6pb2dvc3Y0uVLkDUb2yEU+wfZtrlDGoEu5rhEVlQFQHKybfnr6hqvjSPVimnQXc8/2i8V7SSyUWqKsM5tNs3ykiQrEX+ZsMdpaL7jZcs/iTV/DV2t9c6qFtdQ064RorCR7kqLhTJw9lEHVQquAkbMCp3EqQtamrar4riGqmxOpPqCJe4tBYA28MSxym3khk8v95KzLb5Te/MjjYMYS5rB8S2t/aXCr5sz2lw08lrarKLJGntAywsU3Myxec4BB8x0zsICxrT0ifxhqutvZzahqVnpu+52XcmmRxyvEqWhhI3KVV2LzE7l5/eDYhAEexLca/qmh+FGEl3pd3fvG2pNa267rcG1kdlIlVwg80IvzDPIGcmseHVfFZsrY3p1KGC5t7S8vLiCwDz2bSpOZIYYxG24JJHAu1kkdVlYsejLn+JbrxXfeE9RtHh1JjPpk8dnHDp4ka+DNMhM4wPKfyBA4GYzvkYBHI8obGmar4qfx49pfJJHYtcToYjBK0QhAYxMjrbbA7ARk7rhh8zjaGIVDVtO1O18Z63rdmbt3OmWNvHIttG/lRNcSfaPKOwszoi+YEy2WYZV/kUFhdeK73VLezt7m7Glypcgaje2Qin2D7NtcoY1Al3NcIisqAqA5WTb8+x4our63uLJY7nUrPT3SUz3OmWX2qcSgp5abPLkwhUykts4KKNwzhsv+0dd/tiBIbjVZP3tqttDLp4WG5tWWPzpp5PKHlTDM52bo+Y0HlnOHjsX8aXeh+GZbu+ktby/eJL5Y7NW+zRC1lZmbco2ytIE6gIjbF2sA3mZeq6344tnhEUc6Rw+dHFMbSVvtLx3EqAzJFbSnaY0hf5fJ3eY204xsLL+3dNvdRC299ZWMl3IZ7y208SXCRteai4MYMbGTJNvxtcKkpYBdxarnn69F4RbWbhLu28RXNw9rO0di5kMMdxOYlCRpcLGArZ37JAQcbiWVxc1gazdfDa1kigvrnVhd2cypPCjysVu43yUURDaAM/MIjtHz7Duxl6lq/i+OyhNk2pSaebiVUv5LRo7uRQkRTfEtrKVG9rgf6hOI0O4ZHmaFje+L5RJqNzNJiO9sYksYbBkjkjljthcP86+bsQySsv3SpRt5YDavPyXXjLW/DXibT9TW7jnl0e4ZreO1kYrOAP3MLG1RCjZdCBJMxG0o/Bc6j6Te2evaWkf26GxsPEEUaG10+MCSL+zVjEj7Ivu78xlxhVDkfLsXZua4ZvEEfhO90S5kWCa9+0LeJbl/LheznG/awwpIcAFgQGZcq33Tzeq6344tnhEUc6Rw+dHFMbSVvtLx3EqAzJFbSnaY0hf5fJ3eY204xs6Txxf6rZW9mNKnu43d2Li1tJJGIAGMutvcBRz90oC2QQw2kNh3dpq0XimWcjVbhri00X93dWUUqDZenztzRxlBIisX4bjzHYcKpS58NtMvrXwpd2t1Hd6fPIloqu0W10I061QsodSMqysOQRlSCOCK5/w9Z+I9P8P/AG6yfUop7HwvZ7befT9slzcwm6P2YhkB2AkKQqh2BjIcZy+5f3/i559Rh01746mftiLavZotpBGEl+zSxzMgV5GYW+VMjgeZJlV2nZY1DW9YvLyW4sP7VtdCP2eKWdNMcXMLYuDI0cMkRd8t9lUny2XDNjBDMvP6VdeLryPXNf8ANnS+ttPfEMdin+lT293fBLfguNoAVWVCXO5MSDq3YeGba4tfEvi4XE19Is2oRTRefAFjKG3iGY3CANgqUPJwI1zySW5fTj4x0LwXo1vaLPczSaJG0kT2vlixMZt02rhGbzPKkmYhxIWeL5UxmM9BoGpeJLjwbqk7xfadWh80WKXMMkTSsIwyCTzIbfOXJGVRV24GSQxqmdRvhZE2mseJ7iA3CLeXFxovlz20WyQhoI/sy+YTII1b5JNqknC/eEbaz4ph0y/SSK+a6utPaPRnWwLFrgSXCo82ExEzIbVm80IgJbhcOBh63H4jTwtr2mwWupKlwmpm1t7ew80XbSXV15gkJGUAiMTxkMm4ucCXhK7Dxxf6rZW9mNKnu43d2Li1tJJGIAGMutvcBRz90oC2QQw2kNjyar4vg8WWVsyXcdkXtkUTwNIJYmVPNaYwWzoJQxlHE0aAqpK7OXseCtV8VXmszQ6+kkYNuzzRPBKFhmDKNsT/AGaNNgy4/wBbMThSrEBmaSHUfEp8X2UD/bvsr6hcR3sX2Ffs8MCxzG3KTYBO8LGzYL4fCkxfcePxDrOv2GvBYpbuGA6nY26RrZq9u9pJJCryNKR8spkd49oYnbg+WM+aOb0L/hLtM0JEtbe+tPsWlG5WBdPQfap4rGwEcUhMZZsuZ04Ic7CoYbABsX2r+IrHxHHo0+o6qLBPtO6+g06Oa5ljVLNkkCqmDiSeSPKRsMdVyDItiLVPFM/iPSYLz7dCWitze21lalYY5CgaRTJJA8bqCTlluEO35VUuvzyeCtV8VXmszQ6+kkYNuzzRPBKFhmDKNsT/AGaNNgy4/wBbMThSrEBmaxBda/c+IGtWudSheS4uIriIWSi3tbYCTyZoZmjw8pxASpeQZkfKDaQmfDoN3bfAA6PAl3FenQmzCluiymVoy7RbAnUsSh43nJOd/wA1SWEWu2l9qep2txfS28uq2ISCaxCPdRyQ2kUs0mVDDau44QRhWR92R8q4+pzeLtY0PVtPnOqlZ/D949zGNORQl8PKzbRNtzJHguqkA7gzFZJCMx7njuG+uvCulyafBd6hPGlxIDNZ7ncnTrpVMkRQDLMygqUAywXbzih7vxpb273tuZLy5mvdQtoLCW1WOKOJBctbu7cNlmjhUOWVCkg43Hece58QeKxcaja6bcaldWkKWbrPdWAgulic3IeQIsDsSZI405g+6CdoH747nhu88WapfxJfS+RYpaQyyTG2dHdxPcAoBJFH8zIkPmMVXH8CL5gZc/w3N4ui0W1ubgzxLbS6bbR6aunJEpjkhtVnZgFBCoZJSAm0IyMG3KNq3PDWs6/P4qsrHVJbsJNpk9xdQz2axLFdq8AeOJ8AvEnmEK2CDuz5kn8FjU5vGDWV3Y6cJBqUD3s63BjjEU0TJN9ljQsMFwzQgggY8lyxwyGTH8Valr2pXA1HQ7XUjb2aM9nv0543W6Flfhj5ciByMtbAEgqSQBk7hVzXrzxjpf2m1tZZ72P/AEaaW9S22NEH88SJCEilyqtHAApjlcLKdzYw62PDd54s1S/iS+l8ixS0hlkmNs6O7ie4BQCSKP5mRIfMYquP4EXzAy83ql740voLaH7PqVw4S3ubiF9MUJDfJe2jGKNsp5kSL5hUlgpG4+c+CY7l7/bupXunBre+vbGO7jMF5c6eI7h41vNOcmQCNTHgi442oGSIMQ20NRodp4itpftUIvo79vDWmSJZmyjhgeSJnMtsG8sLEx+6ATlfPY4IRQvaeHdUm1O3N7M8jQag73NgoiO1LUBFQlwMZcYlCthh5hXB2HHn/ibw/qaf8JdPb2t3ey6jezoC1jG7GIaRIE2MsYbHmP5QOe23qz75Lq+8YaOWisLeeC1+1308J+zTSCWRr24bEqx20zeXs8phgxFg7bWbqmw0mu3njbRxdR3zQ2+q3LzR/YgLe2jEFwluyS8bt6MpbmQByATEcI2xLquu2WtX6PYT3FhFdrNvSEMVszAi4j2kF5PP8xiuGbYjcZaINy9paeIpL3QbrWRfQXUGtwG7NrZRsJGbS0V5HKRnK+aXjLg7QHIyNqlNjwLBrumad4bsb955LeXRA0sclqI1spIhAqRjjcGKu+4OSS0ZKhACtEt14xHh/Vr5P+PiK7uIbW3EWHaD7WR5x/dk7lhH7tVV843ESlwi47674rPh6EtcXy3H2uRQYtOufOeMIhBdzYFRhmYYEChhtw+Ufdsfb/ESaxBCX1UTCW1SG1ls42hntysfnyzzRoUSZSZ/lWRATGmFYMN9fwsPEtn4Z0bSWgnimaLT3ikeFQlvbJFALiKTIysmUlABBP75cHCuY4/C3iPxDHqUn/CUmS3idESSN7eYrFcvLHGkcTi2RNm6Qr/rJiflIYgMxua/Brtv4h1/VtKedPs2iW8sMUdqJPtk8b3TLDkg5XkBlUBzvTDL/Fn69rniXT7XWrqGe+S6tor95IHslFpbQRxStBNHKY8PIStuSvmP/rJMoNvyV9W1DxcbKSa0gvrmaDzJY5J9LQSQXX2O83RxrtP7tXFuFcF9xkZfNkBIrY1C58S6feS2ct5fTWJ+zzXGo2mnq8sIcXAdIIwj7lDx242lZHVZWLMeGWuLjXde+HMMk3zauNbRTm2BEXlamFy0aOflRU5Ac8KfnP3qr63qPjHT2ezt3vpbS3u3i/tEW/7+YeVA6ZEdrMpXdJOpKwqP3SgsGyH6S41HWo9I8PX4tpJ5HdG1KC0gILhoHAVUlwyjzjF97aVHLlVDEY9zqfisv/pcd3ZWsaW0F9NYWgmeKULOZZLdCrtIjN9lAJV8K7ZCsr7MttY8Yy22mpcvqsN3JLcCeG007y3A+0yKnzNBPD9xV2hpIwPvNI6uGXYjg13UdsGsPPdadqWoX+nXFkbURpFZ/wCkeXIWUB9xCRqH3BSrr8pY7zoeGba4tfEvi4XE19Is2oRTRefAFjKG3iGY3CANgqUPJwI1zySW4/Qbvxjp/guztrS2vjcW+lQpcQXVl5IsyphVfJIVjIxhMzt/rSHRQY1J8kyQ6j4yuRYyxJIblXeBbqbTJHdEa5sQfM3wQEja05OxUBVAcloyw2Lq98UWl1LZedqUtvbXEkdvepYJLLdyeVA8SSKFVPKLSXCs48oDylBkQ5LWI9W1e68AeGdR+0XaveW9vJqN5ZWomnjVoCxeOIIwJMvlqQEbCuxwMblx7PUPHk+nareXT3cE9jZSXMNoNORTczrcXYWL+P5DGkIKozMQUKyDJL9B4ql1eDV7KbTLeRkFlOJriK2ErwqZ7UMUO0neIvOdU53mMfK+0CuTsX8Swaxqd7FaXxWaWZvts2mqs5g26anmquD+88pZmCADc8fMWV8tdRxq97eeHLqa61mSztddIhmewCzTQtauN88flAxgSs8WdqDY2487ZK2NcM3iCPwne6JcyLBNe/aFvEty/lwvZzjftYYUkOACwIDMuVb7p5vVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8vk7vMbacY2dR4rtriXWvCdzFNfJDb6qTMtrAJQA0Eqhn+Riq5IQtwAJCSQQrLhvrXi+XTtJWztrt9QtreCLVVns2iR7pri1VsOYyGTZ9qy8QZVHzf3KuaofF1pLqr2+ozzrp+lJeQpFYJi8u91wxhXgny8LGpQEyYKYcHcXz31nxwniPUYYbbew+1C2tZYZTA4VHNuVcW6orMViLFrkj5nGFYhU1PCIvpfFWtXVw2pT2jWVnFa3eoWX2aSYK9yWBXC8qz4+4hxt+Ugh35/w1deK7Hwnp1okOpKYNMgjvI5tPEbWIVoUBgGD5r+QZ3IzId8agohPlG5b6r4ldp11G61y10xbuRYb230hWunQRW5hBiEcnyvuuGZtgwygHyv9XWh4dOqReJ7/AMxb4aZdXaSx3UtrtkunFjbr+9BRfKX5XOQoy6lSY9oSSPVk1ex8Z63eWUupMJ9MsQuy0Ekcca3EgnMbCM7pY42LqhLFjIflcBVGppt7rr+GdXmtknvbqLzP7KfUYBby3WIlK+amI9v70un3Uyqqe+45Z1G+FkTaax4nuIDcIt5cXGi+XPbRbJCGgj+zL5hMgjVvkk2qScL94WNCuPFd/q/k3kklvpqW8UonltwkswE9wFGCoCvJEsDSA4KZwEQvlMtb/wAc2vhywnLz3t/e6Utxcq9mIhZyK9uGChUYiTy5JmKsshLxfJHjMZz4ZfEmo3ukXV3az3LQSmHzJLGQsIzeacxMnmW8Q3ACZgyxqAqZB3IzV0HhjUfEtz4htYtS+3G1fT5ZL1Z7FYo4L0PCDDHJgF41BfacEMCx8yTHyZev6x40gluv7OGpGcPfB4Y9NV4YlSCc2hjk25cuViZh+8+fCkx58t9TULnxLp95LZy3l9NYn7PNcajaaerywhxcB0gjCPuUPHbjaVkdVlYsx4Za9nJrth8NGlaO+s9TfVZHn+z2QeaOOTUWMjJF+9B/dszDG8Y5BbgkuJtet57dp9X8Rx6Q3mkXkNhDPcylkt2iDQpbFo1GbkEFAQU+YjcooW98czaFr93cp9ju7O0Mlrb28AkeW4NjGxRAQwaNZmkxguWb5chUIc1688Y6X9ptbWWe9j/0aaW9S22NEH88SJCEilyqtHAApjlcLKdzYw648cviyXU7fVIdN+1aibSNUaWzdUZ449U8sMZY4yjEmAMwWPPmcbVcA9Z4UvdXudZ1SK7m1K40uO3t2srjULAW0kjFpvNyAqkkEIOUTAC/Kc73y9Mk8ZSWOk4ur64uF2Lq32u2hh8ubzrTzEj+RA8Yj+1YddwPOGJ21sWt/rEl5HG7znShsAvPsbpcs2LUoJEZAPnL3G8ogVAOsbK2Ooor4Aor7TsYfFEfieXz5pF0v7a8jZiSTzY2jcKBIZQURSI8qsKndgZfLyt0guJTZwzmynEkmzdASm+PcQDuO7b8uSTgn7pxuOAS9+2eQv2HyPO82Pd5+dvl718zGP4tm7b23YzxmpCJvtCMskYgCMHQoSxbI2kNnAAG7Iwc5HIxzJRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRX/9k=",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAANOElEQVR4Ae1ai3LjOAxrd/b/f7lHAHxJdppE+zrNyG1liQRAinSs6/Y+vz7a9fn1Yd9vXO+hU/jzg1HeDpcCfaIc7jIZbB7sR6cqjcHyhxZvVfVJDu9pjRv+/MCXXRwwftLwaZfMvHMFJ5A+kKq5sOYhpBhUcKOrlQCoUDMdV6E+JGiQM5aF8vyCzJyZFRMqNnGm9RPIdn3lY8AJngMjZRGxJgQm/6GBSYaTeultDKd4kBAqxQ97xuYQEYncFtRlEQuU9EgABugnO4LNG2579+mXfeDA5lIjdqcA+SFISE6ICZ4YbhpuXXFwYIF0kfr1Iq3FwrSlhd2KNLCN9XDDKruRwDAgY/vIaLlXCgMlSGJp90GMbhnnD/w0P/BBwFwKGlnKBldsdGZfN0wFPAD6Qal8z1DD+5XF60KwGobOxNKC0HZhzgJ1o5k82/IDjKvhBu7HV/NkXmbTt4KQTton4d4aBYuQivNs/Bb9rfOZ8l/z21sa1X/xyraq685ygeZ8KvcoZryQU0DACT4sc5ETkW3ZLDm1DWeiGeWVyRrrqfK97L11EruCrpaPD3bY+oUK2ICyxLmnCVcw40sojEJi5EION4bNjLRTU4xCIw5sXRPLGUwAYA4NceqRgBi4SHYBzP0r3ZjYOcxPng1xmuYnUROM7acfmWZ2Mcx46BErY5CYShMJiPuDqvcXRQYw7IDacxgz5O1G2OlFlLhkcHsiObFBb2ko4vJ3XSxlxEgRmaNiMPNViGR0aenGjqsadCDmV00hFDEBboxbBIz1cEf8BFwm47GExAzC7Y0itUoJM/VN5dKNxFmdISZOMSde5Xcfpqw+uybYIRbnMaBt2Cpj375nS5KzoVxQldlnXHgsm+d34bTfFC0eqFEByvRFzF263bQTBGImANLEQZoR7GbTBpis07JF+p9MHyaIErxyxW9LKItdI8mMbpfXxzvbALhZ3HEe2rqjz013TBD56aG0/wSL2cRQMmmMDd/kSNMc4BHut9rfDPoW/Ad6yLPQUrZioRI8IbWk6cacGE3IGuiTxevRz16YKhyyCBCEwgVQzKkpiUgRH+UkYs6rZLG0FaV5t3PYLMnBjK8ZflQ4w1N0NcNPjCaGdEtOPLQLajOJEXfUpRzyY9Y2UoqbrKzSKfn7MbiSwgo4v//kG1Lv3usnZBBU0oMJOjSoNtdnKyyK4twwekOkoK3EfqNX+Sr2QHoPa5wzybX0W7oRHXd02K7mTd48yTwHh6tHhwdfLb7Tl0Koq3TJLGLCrDQ2jwIlcp6EoNlDBHd0WN9JQB0c3VVhjp7Ub6WyIgf+GED0JlL63TiEi3gwth0bHhRenGjgBvCYuwu710cpDZCxBRA2CS7ujdXQa9PfqbWWwSNWy+z+WGJZo7alMlmmZW9TcWrW4DbVd3nbrAFh7UvN+9h4NeVrupY9s/sNN+ye03ra5/ztHEa97Fyzm063Ko+72A0/+by4PClVezhGYMSAA18K4FYa0ba0KjSSQAZlz7RCwfy2EcoRTWyQMg4laMWA75bhp72l45AzF+TwzjAYrnCB5SXTXZB4u/D1ETwAHCxUV3FNaJMMRExgJNZvCNiEEuaqriB1FSnE5AI3firDj88fvhOEYylwd1u44p4ZFATougD0J0FGGIId9zSUqxRkC+iXxLQs4XAjmuYYOXNGCTpABki0Xw8rgeowgb6EaPLLVuIsdMvG8pmck1zz5rRTsImMWMIN4saydAY0i+UL/xcPMPzfcOPcwvYMryVOMJC144BkMuYcgHQEqpyJginIkG6nNyPBb5cH5Zx2ZoBIvFzBlmFpDJiG3TLDCiuJ3zXGZlb0foX7QjwdSyyZ102FTWpa02KTW2MDZMVlewYfFO0N8wK+gum9jnWjtWkhiWnn8PQAjMBfWr2r/Cb+LbhtOAqKUmHOksU7sRYoGry0BCkY5SFxYPmCuh4LIE4VRtbbkGqVuRgagxJs7IxtGxeQrQRTtBYKb2n7zMDHC3N7d/Hc0ltlWBjG0Q7RGoFIQTJcdBYsF5rgfAVeTvALmaIhnfd+giM8NsBYmDCgy/f5l/4gnh855Pb0Gt5/YnggeLJ4rpM5DLQ5yB3TMElueNo8bDNDgW9ruTqgz3UOz0k2nZvpgFZKkuRmfZq8XA+0dPuE2c7Gy7lCQAqObsmbUxkNeXXeTz8/h2D6s6orx0IdspG1DDzWmstDJ8Ni5i6Wnp6g+V0YG/VdBDkCjdX4l2F6ZMY06FU3WJRppUd5VYL8l4fGadOX6f8W+EPvUE9Cz8WckVmBknOEZAvBcdcdbpZM9Ch3AwtTxI/1o7ve+MhlkI6k8MvDkPQDnVcwD6i/zfxiDt/DvtjhVr1+vlmRcJDpy/MWtFCoHKpZds1gpo8jV5j5wUiOIyDMOHL6KD4UCKMUXVSxGe0pKRgCqLGVDqD6Aubj8ycKwgNLaxyQ+GDyw6lPaHkDavaGgozOQdoDPoJsJZyUsQiWB6Ojybpmg+GlQ1WNkySW+o8Ik449UM/qEEld/rclpqUikYYY1+vWBtkOHaW6p835ouXb1ODoWoycymvTnNDsgWBMljuGW6QzsIffhwMeSOrlIrz3d2V78T1hO8tQ1iBhGyM1c+L6am5nXQI3w8D2DetRchArgrhA8t6SQGa8VOAcWxkDAXZIaJJoytqKZo9TkNGbMMXFCNPA8lAtz0gi2LGOe+6jVN+cpdKbvJfgo/i4ekkgQfnrYSsOncNzkHCf3DkpIIe/FSfU+Lq8BjG4vudotq7sIFqrq8q9pVLJDRPYhkm0eZ5Pwb3hx6f0gcAN4wHyqfkbKfx92A811pcnG0rNLxSGjcGdPxoJkNNG/waD8IS6xaxw8EvTiOiOcocC8dDhwY0x567uHiogR4daFCDxJRvh9ApiLy37RMRxhQ9H/AwO6fYxWIHHGpI8Kgmss29aKoIT6IPFrjo9w5pQM2hOz7SCVONSLWxISnBBbMPRf7wl5SZjdIjWRrFGvK1CzJGWhos1ahTd39HdY/NUUN7wIrM0B1yhueq+mOddOC1t3s5heGQPsGs75S73AQ/czfZcpd1CX8Ig3dGIygCF8CKGSNPFtHByRAyujBMb1nGFUk61lANwZjUWu+Edhxu1MVhwkqYkxHKCdGPHE1LPgPDV50xE6h4OKAsN2UwgY8gCSI/Q58nB5KFjQN0vfoV7r9itF/WLYU6+H0tVmdJETVgy9D6uPg9bv8uPVyWLDgldcQ/wvA57Q18habHJJeWLARBlItG+4THcN6ur6ggOf9xH779YVSbasIpmox1eHHnHSYbDRt442ODHF6yCx8SN8Lsbd8f7Np0GMnD40l0LrkKamJAiUucxCMkiGKs6qs1L5RIDwYAw6P+XBgFFsAEu1QMLm7X3id6mdNOJwVjBI7247jBrSAjIFUXKrqAhapSUlqiWJCmcjcRYOFrlwth+Wr6UQUbxlsacl7/YsNMwuYM3bbCcePHxwmRgpMcBQnEMTvMwYfHD6wIhWnraXCPP0xsxQKjAf/GwRR6FCGcxrqKyRHgI4IrsaiJ784Sh7rOIeZoppi4dmWSkfOZKcJqFwqirFPVI4+nQD8rge04V3yre1ixS6tmKBoyaBCcMsAKe9QSAUInEP3oHL70hQDTjhsWISBVXY0sTo1z091BBj/o54P3bqwKv4t7P4D3GNY+rZVYc2jU753U9ifIMDZjBf2G9dA6Pj827Wc4leJf/a3jbMA8o++QikX6qwYQ1v9PPdGHSZx0YQ/iCH/8nCeEJosgT3B9y59+HLQ8+qp6LnjzkVj96lPXIA66UhLSmw+2L75Ll62UGpOjs+O1rnsP95Vb5KAmLyIm20ltTyLeyStkr64VyXUlvWvgZ7tvoc9dqeXzZFRFmZDoCcH+3/3cMD4fYofEi917xLev492FrmvfNJjp1h1TopCWRmBjSH+hv2pdpGV/fZhnUE/EnJ69k+Evxo4ch8ve3GJF1/+MbHsP9+9XaOfzv817O4Gx4uXSbEE+HN2nUcpqnw8ul24R4OrxJo5bTPB1eLt0mxNPhTRq1nObp8HLpNiGeDm/SqOU0T4eXS7cJ8XR4k0Ytp3k6vFy6TYinw5s0ajnN0+Hl0m1CPB3epFHLaZ4OL5duE+Lp8CaNWk7zdHi5dJsQT4c3adRymqfDy6XbhHg6vEmjltM8HV4u3SbE0+FNGrWc5unwcuk2IZ4Ob9Ko5TRPh5dLtwnxdHiTRi2neTq8XLpNiKfDmzRqOc3T4eXSbUI8Hd6kUctpng4vl24T4unwJo1aTvN0eLl0mxBPhzdp1HKap8PLpduEeDq8SaOW0zwdXi7dJsTT4U0atZzm6fBy6TYhng5v0qjlNE+Hl0u3CfF0eJNGLad5Orxcuk2Ip8ObNGo5zdPh5dJtQjwd3qRRy2meDi+XbhPi6fAmjVpO83R4uXSbEE+HN2nUcpqnw8ul24R4OrxJo5bTPB1eLt0mxNPhTRq1nObp8HLpNiGeDm/SqOU0T4eXS7cJ8XR4k0Ytp3k6vFy6TYj/AWPx9JsjP5R6AAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g1.to_PIL()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf64fW9R8Sw+IfKs/twj/tC0jSGGxWWBrJni82ZpCPlk3GVCAxIQbvLA/einLe+N4/AumajM0n9oXKW4u4oYSJLVPKYs+BBIxlaTYGURMFB2hRtaU7ljdeILvwJHdr82spmRYzE0RuBHKSIm86OPa0iLsL7FALllUDArLk1jV723ttSN7qVrouoPNPBNpliLmeOMCNbddvlSHZKolmJKAqWVCRjDSWur+KTEtnf2s8Or3F3YyokNqXgjt9tubpfNAaNcEXQAZ9/TbnKZk8TxavB4x0/UdK+1zTpo+oR29uIA1u1x+6aNJH25QOV6l1BMSgEZYNxert4n17w7c291b3175X2gxb7OYTZfT71CCTa26lSxjAAUnc2CfmUV0CTeLoo9fmtjPbQ2EVzd2lpHpyD7ZP9rvCqZ2/MrosJYKN7b0YOpYl7i6zr8fjPTLS4lu44LnU7iC4gezUQJEsU7W/lSkAsXVFdsF8MMExfcaxc/wDCST+JLmKPUNStrGbUxZp5VrCVggFmsxmRmiOSZlMZZiy/MVADYIx7TX/G1xrekveWklnFcJZs9uLWYxFZEjM28LbuUdXaVRunj27ELLjJex8RdOvrm/eS2OpSpL4c1W1WC2tvMjaVkjKoxCEqXwSMsMmJQOrBrmoXPiXT7yWzlvL6axP2ea41G009XlhDi4DpBGEfcoeO3G0rI6rKxZjwy09CGv3Xijw7qWuNqUU5t9Vt5YhZKkSkXKeWr7Q+wNGoIPmEHykwzZbfH4o03VJfF2pPZyXztcRaO1uGs/NtkMd+S5LKoJ2AhypcHEjnoBskvbnxHZapeRlruZLK3uobfUn03zZZd32F15iiOCWeZQVQr+7DMknlnNyKbWbv4Y65JPHfXWovaXQgjvLdHMh8shVWPyIiyk8bZIgSd3DKVJk06/1q48eXEEs+pPp0byAKtoYYAAMDd5tuCRn+KOdyzYYKEJCGo3+tHx5b2dvPqR08PGHjtrQogGMnc8luyOO5ZZ4zj5VUuvz5aaj4uuLaKOO41Vbqb7OuoltPRRYTNcwK6WxaLEsYRrnLnzQBGh3c5aS51DxpaWc5txd3ckiXS5mtlXyVguo4VkQpGfnkgaSXBWTcygom35DIs2t6x8LPE8N9HPcXjWl1DbKbeVZpAYOAQ8EO5tzMBsjAxtHLAkx6ndeK21C0ktoZIr/fJDKx08TRWwa7sl2o+AzxGEyOTuBbDkiMpsisXV74otLqWy87Upbe2uJI7e9SwSWW7k8qB4kkUKqeUWkuFZx5QHlKDIhyWkszqmmfDfwrAi31nstLaLUJLa1825tkFufuxFHJbzBGhGxsBmOBjcsen6dqepeL/Der6mbtha2WoCMy20ajBmjSF3ym6OWSEhmUFDlSAqjepj8USa7da0tosd89uuq6e0cENkHga1WaB2naXgrIJA6kbj8i58vBMop6HfeIxdeG7KT+0oIVsrQXNtbaf9nSNvKUurb7Yx4BznbNHgfIqb1+ePS9b8cSLeLfRzxzeUjXCC0lc2x82MSm3JtkRtkbTELvuCxRNu8bt5ca14sWwuB9o1VcagyWrnSnjmlthBCVdiltOq5ZnJDRqSxOPK2NFVyTVfF8HiyytmS7jsi9siieBpBLEyp5rTGC2dBKGMo4mjQFVJXZy+xY+EdMGs3cq2UdvbwvEvlxLIrXcqtFOJ55CAZnV1AU5bGZNzMXZUNc0SaGPwnp2ieZaQWV75aypGZvs0K2c6Z+bIzghQzZG4rkN908/rGseNIdJvfs41Jby3t7hLDyNNWb7ZJHNOhaYFcKfKSGRcFAzSHasgxHR4hHiu48K6t5jXd39ufVbFbFLIDy4Al0YX4G8uTHEobIVlkUbSx3nrIbrXT4e1edbfzNbj87yLSVAsIkCfu0R+PMjPyneSCSzZ8sgxpyc2s+Jo01NYLvWZLNUtRY3Nxo7Rzylmn837kDlSCqDc0AG0KNoLrK1jSG19fEcN1PFqS29zZWKXV3NArzbhPdEJtWNAEO9AzbA6Iy7kQszx09L1vxxIt4t9HPHN5SNcILSVzbHzYxKbcm2RG2RtMQu+4LFE27xu37ngm7mTUdZt9RN2Lu+vXubVrq1MD3FvHb2sfmlRwD8yA/dOSfkQgouXaa74uuZb97AT3t1Hd6jDDbTWiR2phiadYW80YzJ5qRxEF1O0klOPNNfVtQ8S/YpJdIOq6jHb+ZJa3d/pKpcrP9jvMqsZhTChhbhW2fMZWTcwJUXPE9tr7Wd3Zz3+pTxW9xpd8LmzsFZtxuj5yRoI3ykaxpIFw7jjczg4MaTeLoo9fmtjPbQ2EVzd2lpHpyD7ZP9rvCqZ2/MrosJYKN7b0YOpYl7i6zr8fjPTLS4lu44LnU7iC4gezUQJEsU7W/lSkAsXVFdsF8MMExfcaPxRJrt1rS2ix3z266rp7RwQ2QeBrVZoHadpeCsgkDqRuPyLny8EyiO21XxWYo5c6lLKiQzarFNYBFtHE8PmxW37sGZDEbkZUyk7E2tlhvsanqfjSWSBLSzksZA4EpNss0YWS8txEThjuKwG4EgVhgqxyFMbnYt5tft9I8QwRiS/v7F3TTZbuNUN2fISRdxUIhHmOyZUKMLg8gmuP1STX9Ss9Qja88QSaZa3GlXFrcNpKpcSgXTGZvL8olim1Wx5aHEafKyktJ1niu2uJda8J3MU18kNvqpMy2sAlADQSqGf5GKrkhC3AAkJJBCsvL6cfGOheC9Gt7RZ7maTRI2kie18sWJjNum1cIzeZ5UkzEOJCzxfKmMxnc0e98TX3gXWXDSNrCJOmntLC0chfygU3+bBCpO8nBEYXAAOSGJju9T8Qaxr1qNGlvrLSJpYYJJp9MZHjIiu2mIWVAw6WwVyCm7Aw3zqa7anru8Rare65p9vF5sUFxpulC4kumS4mjLSqIJAuY0gcEKisZXIyAAserar4riGqmxOpPqCJe4tBYA28MSxym3khk8v8AeSsy2+U3vzI42DGEsa1eeJtLS+tVvru4Oy2mjuodOYku7TCSKPy4ZQiKI4yC6SkByrNl1ZY/7Q1kQWQ1K/8AEdjA1osiTWmmpdTTSl5PMEoW2bYqqISmY4mIc7huDKleS98S2kd2kMd9bSLLePpsNrpqtHeTm7uMJcERny4yotzvzHuErsXJyy6jaxryulni7S7tr29lu5G015Ivsm2c2+0qoEp5tvkjbzDyDyHxY8D3er3lveSarJqUhDqsZu4wi9DnaDbW755GcoV+7tYncBTkvfFw0y3BSdZrG0W31OYQIxmmMkIee3UA79sQnkAC4JkRdjMGjTHGp+NnbUZY5dVMNlFeXFmsmmRxm9EcVo1uki7CfnLzZClHJ3ACMrsSxfav4isfEcejT6jqosE+07r6DTo5rmWNUs2SQKqYOJJ5I8pGwx1XIMiyaTL40utda3nt49NF2jPfTJbLtikFtYcxttIkcMbhFLMRwSd4iCEjn8Q6t4svdF1qCT+x7t7m2mtzFN5RtirhCkiW4VXZQhJNww+ZxtVsKmhp1/rVx48uIJZ9SfTo3kAVbQwwAAYG7zbcEjP8Uc7lmwwUISEprJrtn421gWsd8sNxqts8Mf2IG3uYzBbpcM8vO3YisV5jBcEAynKLn/2z45FxaCCK+mkeKNruKewEUMV0bq1WWFJQhJhWNpcPtb5SzCSQr8lzxRf+MtJvbaK0nknjW3EguI7SRkmnLuWR44redhEo8sKN0bEEjzHbLLI0mu3njbRxdR3zQ2+q3LzR/YgLe2jEFwluyS8bt6MpbmQByATEcI0mradqdr4z1vW7M3budMsbeORbaN/Kia4k+0eUdhZnRF8wJlsswyr/ACKK+kT+MNV1t7ObUNSs9N33Oy7k0yOOV4lS0MJG5SquxeYncvP7wbEIAjryeI/FezS7q7W7sZ7i40xJbVbAfZxBM0AlZnflJfNeSLbuLBQD5Y/1o1PC2raxPq17Jqk2qzWowsebF4oQzyBQRG9tHKvXgB5lVd29xhWbU8VaUq6bqeu6fbSPr1tZO9lJHudvNjinEWE5DH9/IMEHO7vgYy9QufEun3ktnLeX01ifs81xqNpp6vLCHFwHSCMI+5Q8duNpWR1WVizHhl3PDeqTXlr9kv3kOoQp5rCWIxu1u0sqwSOMAK7pFuZcAg5BVeBXL6RdeMI7XTp7u51K4k+z6ZPPFPZRqHe5l8u5jYLGpAhVd4AIKliXLLgDoPCVxrT6Dpl14ikk+33tvCpt0tyBC4jLM0hCgq7EEtnaqnagGfmfH1PVfFUHjxLaNJI9MNxAkQEEskUsLBfMZyls+HDGQDM0YG1Cy7cl5JL3xcNMtwUnWaxtFt9TmECMZpjJCHnt1AO/bEJ5AAuCZEXYzBo0p6RP4w1XW3s5tQ1Kz03fc7LuTTI45XiVLQwkblKq7F5idy8/vBsQgCMsta8TX+t6L9pGpW6zW9tJd2ttYtCkTsivIrNNburAEnJE6ED5Qpdfn6DVL6+1a08PT+Hr2SK01K4DS3SW3mbbZreVw2GHyEsI8FhgMVyGGVPP3Oq+K0ikUnUkuIEmj05YbAOuoTpPMii5PlkRIyJbtuBhU+a5DAAbNzwzbXFr4l8XC4mvpFm1CKaLz4AsZQ28QzG4QBsFSh5OBGueSS3Lxa/4uXVLKCSS+LNafar6B9PRfLkS5tRcxwnG6WNInkClQxbd8skrYCHiT+3dftdfMFvfCxn0rUVsG/s8JMVEVmBFh49675BcDaQGYcj7qMLkmr+Ko/FllDG122mM9ssTz2kuLiFlTzJJlS1OyXcZBgywhSqlkC532PGuq+KrDWYV0lJEsxbq8TpBLKss25tyyrFbTNsAEfQxE7mwxP3Ll3f+JUnvRYxTzxp9qOnmW1VftE6oSkUpyNkIbcFkwm/aoLABWuMdNV8Sm8uoxda42iL9m26g+kKt2qEXHmMsXlku3mLAhBiGEOdmP3rU/FNlfX9vHPbXfiC/Sfw5q9qom03Y0krBCqSKIFKFsHGQu7ykxnc2/pG1vUbHWLy21G48ux0nz9SvLzYp8yzZWMKbFGRgmUZGWP2PJH70V0FhomnaXdXFzZ2/lTXOfNbezbsyyS9CTj55pTx/ex0AA4PQ77xGLrw3ZSf2lBCtlaC5trbT/s6Rt5Sl1bfbGPAOc7Zo8D5FTevzmjjxXonhPQrVWu5RLo8JuFFkA2niNrZCI1wWMohkmba+8s8XyoBlDY0ifxhqutvZzahqVnpu+52XcmmRxyvEqWhhI3KVV2LzE7l5/eDYhAEepqmra23gHTL8LPZatcxQPcRQWcrujsm50AEUxjwc8tG/TbwWDDD1bVfGFv4fMlmdZ+2jTGlsx9gjna5nJk8wS4jQIEQRNGCImO7DJI4MVU/EL+JZfDWu2EdpfIs8WoraW9ppqut2XuLkOJcAbMReU6NuQuzk/vj8h6jxfJrsWsaW+iR4kaIxtcGyE4i3Xdmp3HqFMZlJUMuQhOQUBW5bza/b6R4hgjEl/f2Lummy3caobs+Qki7ioRCPMdkyoUYXB5BNV/Cl7q9zrOqRXc2pXGlx29u1lcahYC2kkYtN5uQFUkghByiYAX5Tne/P6YdX0rTjaTX/AIggtBqepGe7g0sS3HmG43QhUEDBopEaSQuEI3YAZRhK0LG78aMJL/UDJFJFe2MJ02G1UxlZY7YXDb/mZkRpJipVhtKNuZ1G1TwVqviq81maHX0kjBt2eaJ4JQsMwZRtif7NGmwZcf62YnClWIDM2fJpOrv4yu0t7vUkLeI1uBcSWQaOGFtMdC8bbApOcxAksFKx7lJJ3msax40h0m9+zjUlvLe3uEsPI01Zvtkkc06FpgVwp8pIZFwUDNIdqyDEddJ4Ztri18S+LhcTX0izahFNF58AWMobeIZjcIA2CpQ8nAjXPJJbm9MOr6VpxtJr/wAQQWg1PUjPdwaWJbjzDcboQqCBg0UiNJIXCEbsAMowlFpceK7a4u5rm0u7Ge7uFnm+x2YnD3YsrNUiwSR5BYThn3KAYwPNTqbltpeqP4R+HdrAJ7K6tvIFxIbfc9sBYTIxKnhWydoLAgMVyG+6aesax40h0m9+zjUlvLe3uEsPI01Zvtkkc06FpgVwp8pIZFwUDNIdqyDEdHiEeK7jwrq3mNd3f259VsVsUsgPLgCXRhfgby5McShshWWRRtLHeeg1CXV7jwL4lM1vJJqn2e6VLQ2wkjDeUdiRgqRMhG05OdxZgVTmNJPDNtcWviXxcLia+kWbUIpovPgCxlDbxDMbhAGwVKHk4Ea55JLZ9nqvi6XTNMgnsJ0un+wwzTmFATMkim9LDOFjKbwjAAEo+0/NCXp22q+KzFHLnUpZUSGbVYprAIto4nh82K2/dgzIYjcjKmUnYm1ssN8d7d+ILrxC+rWdtfCzEVxHaytZMskNsz6eJWEbLu8wbbp0V1JYpwrrgEitdUuvAvxAE1vfS3F59o+yma08qa6T7DEkbeWP4m24IAX5sjYhyil7q3jFNOna0W+kjEsIuJ5bPyZrdyJfOSJFil3Rqy24DBJ8iR/3jAb02EudYvvA+nTXMPnX51C1EgNo+fLW8QF2SSJCGEY3FvLQAgsoUAYPCk/iX/iSnWp57lr/AEo3F2JLRYVtZ18kKgAAKswkkLBicshKhACtc/e2/iXT/C+pm2Se5W8u9YiXTzYLIiRsbuWOQgqS7M6xqM/IyyKuwkhjuadf61cePLiCWfUn06N5AFW0MMAAGBu823BIz/FHO5ZsMFCEhKaya7Z+NtYFrHfLDcarbPDH9iBt7mMwW6XDPLzt2IrFeYwXBAMpyi5+ma74zMGpf2kZ7ecWhdkFhcTeTPvQbISloF2/M4zuucYVvnVX32IV1m+Twnf31zrm6HW3Mu61Rvka3lRWIFtG6x7jsLPGmPMc5x5bjU1oa+3jeKDTGktLS4S1invEslkwmy+ZxvI6hhDjJIVnX5SHKtlznxL/AG1p+oot9Nfwafq9pbxPaqIZ5Y5lEBmYINnmrGrk7kUlBt2gkNHpGr+NrmW2gjaSQPeojSX1pMNkbQXBZnJtbcFAyQkBQCW+VnUOuLl3qPiyy8aW9kHnfT1lt4o5Ht3dLiIhBJJJ5VqyrJuMgH72JRtUlNvL5d1J4j17w14ttprjWZd+mTi3VLHyUdiDhFjltUkBYAqFV5sAsC4bYzdJ4wguLzwTZPbXeqloruwneSGzDXLos8ZZmi8okMB+8KhBgpgjGVJYX+uv4oWGZ745u50ubV7MLaQWoEnkyxzbBvkbEGV8x8eZJ8q7fkz1k12z8bawLWO+WG41W2eGP7EDb3MZgt0uGeXnbsRWK8xguCAZTlFk8Far4qvNZmh19JIwbdnmieCULDMGUbYn+zRpsGXH+tmJwpViAzNn65feI7q68SW0H9pXUCWV2ILddPxAzCJgisk1t+8G7A+SaTe2CECFglO6vvGGjlorC3ngtftd9PCfs00glka9uGxKsdtM3l7PKYYMRYO21m6p0lo/iiS4he5vrsR3mp31kY0s0X7JbKbgwz5Kkl8xxAM3yFXUFCx3GPQDqmn/AAj0ZLZb4X0NpbxTCa12zwIGVZtsZQbmjTfsG1t2xeJM/NJpl34ou/C/icqbttQjeVNGkurVLd5F+zRmJip+XJckndjnIKpjYuf4uvfEer+HtStLTSrtbe6t7towLf55bU2cqhXByyymcxsEwrFXQYysoWn41t/EsllfaUyT6tG1pJKJ1sFO2SWzvlaOLap2qGSELyX/AHu1nYOBWhaa34nh8aXEepRzppyS3G6FLSaVPIUOYmjMdtjzGCxkgzvncyhAxCrqeK59StdSgnsEkULpl2PtEdi1wySGW2CqCiOylhv6Kw+XcyOExWHpVr4i1fVtHuJr7VbRhaX6S3k1tGXVTJalVj3QRBclD/rYdx2yYBG1wLrnjG4/syeS3vra6nl04zWcOnboFtpPJM7s7DKSB2lQruJVFz5Y/wBaNjwpP4l/4kp1qee5a/0o3F2JLRYVtZ18kKgAAKswkkLBicshKhACtV5brxiPD+rXyf8AHxFd3ENrbiLDtB9rI84/uydywj92qq+cbiJS4Rc+y1jxZLp0Bne+CiWZY5YtOeSaWUCIwxSrJBCPLYtNl1SJQEQGVTuLWNC1PXbhtL/t+91yyunitCsFrpQeGfdFGZDO/kP5TeaZUYbo9qqDhfvHP04+MdC8F6Nb2iz3M0miRtJE9r5YsTGbdNq4Rm8zypJmIcSFni+VMZjNiy1jxZLp0Bne+CiWZY5YtOeSaWUCIwxSrJBCPLYtNl1SJQEQGVTuLaFmdU0z4b+FYEW+s9lpbRahJbWvm3Nsgtz92Io5LeYI0I2NgMxwMblrtrPimHTL9JIr5rq609o9GdbAsWuBJcKjzYTETMhtWbzQiAluFw4Gx4Ztri18S+LhcTX0izahFNF58AWMobeIZjcIA2CpQ8nAjXPJJbl/7W8cyXFpHA18sksUZuzPpQMNtdG6tVljQjaZIUjeXB7ruIlcjMepdXvii0upbLztSlt7a4kjt71LBJZbuTyoHiSRQqp5RaS4VnHlAeUoMiHJaubvV9E8BeBLS3N3aTypb2t4kdqJZ0jWzkZ8RtzvUx7sYJyuNr/caNL3xKLp57iO+t7WSK3SXULfTVe78gS3uxwnlsWkYC13psOzznOyPJK2LB9UXxWt1BJqs+m3FpZRT6hdWHl3DFZ7rCGIxoFXLKrMEyqMpwAxlTP0vW/HEi3i30c8c3lI1wgtJXNsfNjEptybZEbZG0xC77gsUTbvG7fqaImv/wDCPeMp7ZLv+1J7iR9NlurZYHnIs4VhkKN8oJKjOQoyDlU5Ra+u+Jdak1azk0t9SttJubiG2JbTCJ8iG7knKRSqrsQqwMCAwyuFWQ7kbQ1iLV9X+E2tQzfa/t8tldrDsgAnnjG8RboyvDyRhNyhVILsAEOAKdhFrtpfanqdrcX0tvLqtiEgmsQj3UckNpFLNJlQw2ruOEEYVkfdkfKsngrVfFV5rM0OvpJGDbs80TwShYZgyjbE/wBmjTYMuP8AWzE4UqxAZmz9OuPFennUINNtLuS4F7qkkNndWYjtmiMlxJFKJSRudpjGuC6/IxPl4AlNz+09dEUggvdcm0jzYhLqU2lBL6LKylxHB5A3KGW2GfJbiWTn5cpT0i58Uabo8NrJHJb3es3F5BCHhSM2d013NIZdhZ94MLvKFLFcWwUMxlBrrNfutdtdRtRpNv56TWlxEqsgMa3RMRhaUjDLGFExYg9BgZcop4/xIviLXbXXx9ivoLebStRS0X7HGZlxFZjyc7WPzyC5XAPzgblJARhoWEWu2l9qep2txfS28uq2ISCaxCPdRyQ2kUs0mVDDau44QRhWR92R8q6Him/1221GRdPe+TbaK+nx2tmJo7y6y+YrhtjeVHxD826LiRzu4ytN7vxpb273tuZLy5mvdQtoLCW1WOKOJBctbu7cNlmjhUOWVCkg43HebngXUdbvvt6au88kcflmFrm3ljkyd24Za1t1K8LgKpIJbJwVAw9HHivRPCehWqtdyiXR4TcKLIBtPEbWyERrgsZRDJM2195Z4vlQDKGxpE/jDVdbezm1DUrPTd9zsu5NMjjleJUtDCRuUqrsXmJ3Lz+8GxCAI6d1feKdYg0r7dZ30U0t3pU8llDp5MCQ77eSWVnPzJIs3mIV3MQi5MeMyjY8caj4lsvtv9i/blkTTzJYJaWK3K3E/wC88wS5Hy7FEbJ8y7mYgCUjy6je78aW9u97bmS8uZr3ULaCwltVjijiQXLW7u3DZZo4VDllQpIONx3mx4RF9L4q1q6uG1Ke0ays4rW71Cy+zSTBXuSwK4XlWfH3EONvykEO/PyeIPGiTJbxfa5LuTTJJ7iCeyWIQXCzWyzLAwQtIkSSOUISQMT1nPyroaHdeL9YuLO1uNRu7O1KXRmuRYMJjtNsYwWmt4lD5ebBEW0oCMFgXXPeHWtR1WLUL2DUluL6y0G4ktVsyIInS9DzLu2ZUx5LbWfIEjk5Cjb1nhu+1m91GS1v5Pl0mI2l2+1P9LuSQVkwP9X+6Ecm0ZH+lbc5jNY8uqeKX8R6slh9uuhBFcG0ga1MFs0ioRGrmWBcruwNyXDFjhgqoSEx5Z/Emr+Grtb651ULa6hp1wjRWEj3JUXCmTh7KIOqhVcBI2YFTuJUha6D4h2Wo39ne21lDOVk8P6ku+C2WRzJmDZEGKkjeA42jBbGRyoI1NZur6HQdMktLnUhBI6C8vIrLfdpF5bEOIfLPzmQRqy+UcB2+VcZXLjvfFx0y4ASdpr60a30yYwIphmEkwSe4UgbN0RgkIK4BjddisVjfH1LW/HEOp63HDHOjJFe/ZoUtJZUwscht2jIttnmMViJDTuDuZdgYhV6jxFZ6pP8PPEcVyftN/dafc7Le2TcsZaIhYo+Az/UjLMSQFBCLl6hJ4jstdtoJZJL2CF1uzetpfmrC7216HVEQbtitHDhQxk/e7S7bxWh4Hu9XvLe8k1WTUpCHVYzdxhF6HO0G2t3zyM5Qr93axO4Dl5J/E2swXcbS6lMVvbJ4Gm09o7dCL2Ft5ieCGUAKpbaJZQFDb3B2s2xdXmv2l1LY3l9rMem29xJGup2mnLPdTHyoHjBRYWUoTJcKWWMAGJAWBzvktdX8UmJbO/tZ4dXuLuxlRIbUvBHb7bc3S+aA0a4IugAz7+m3OUzzdnH4j/4R/w5pc1rqSC3fRh9jWwzELdDbNJK8hAKSrKHUruJCLkxgZkFiw13x84lWW3kN4XtzJbSQShI3NzCsiJJ9lVBFsaUM2+cgAMrEKWPomnXGyV9Mub77XqUESXFwwi8sbZGcLtA4C5RwBlmAUbiSdx87upPEeveGvFttNcazLv0ycW6pY+SjsQcIsctqkgLAFQqvNgFgXDbGbc/tHXf7YgSG41WT97arbQy6eFhubVlj86aeTyh5UwzOdm6PmNB5Zzh6dvqvitrVmB1KSc26tqay2AQWEpliDrafux54WNrkj/XZMUfLbsPY0RNf/4R7xlPbJd/2pPcSPpst1bLA85FnCsMhRvlBJUZyFGQcqnKLX13xLrUmrWcmlvqVtpNzcQ2xLaYRPkQ3ck5SKVVdiFWBgQGGVwqyHcjdR4b1Sa8tfsl+8h1CFPNYSxGN2t2llWCRxgBXdItzLgEHIKrwK4v+1vHMlxaRwNfLJLFGbsz6UDDbXRurVZY0I2mSFI3lwe67iJXIzHqXV74otLqWy87Upbe2uJI7e9SwSWW7k8qB4kkUKqeUWkuFZx5QHlKDIhyWr3enXw+D/hqAHUrW4tE0p547W28ydQkkO/MZRjlMF8Bc5QZyMgyaxrPimPSbu1sIr46vay6hM5WwJRrcR3BtQjlPLkbJtflUl+oYcPWpPb+KJ9Z0+1bVZLSC5t7me7e0tkkWBlaARRRu6EZwZPmdTv/AHhCr8oj5+01/wAbXGt6S95aSWcVwlmz24tZjEVkSMzbwtu5R1dpVG6ePbsQsuMl7nhuTXb3xlp91qsd8fL0q5W5M9kIo7e6eS2LwRuMb4xs+UkNkBiJJOdkZGr6ZrHiUx3Wspb3GsRSTXEVgJXitzaLh4AIiJD5qJEwAkKquSAcudDWItX1f4Ta1DN9r+3y2V2sOyACeeMbxFujK8PJGE3KFUguwAQ4Ap3k3iLSdRttN04332e3u7G2iih06NoHscwrLMzqqqkm4yIVU8IMiJR+8WNdU8R/2JdTRXOsy68LdGuLCfTfLtbZi6CbyJRbnzCil9mGm3AZ2y95Fm1vWPhZ4nhvo57i8a0uobZTbyrNIDBwCHgh3NuZgNkYGNo5YEmxqFz4l0+8ls5by+msT9nmuNRtNPV5YQ4uA6QRhH3KHjtxtKyOqysWY8MuXZ6h48n07Vby6e7gnsbKS5htBpyKbmdbi7Cxfx/IY0hBVGZiChWQZJfc8HjVIL29tr6Ce2tftd/Lap5PyTBryRi7uRlWwV2rwCrFgXyRFh6/rHjSCW6/s4akZw98Hhj01XhiVIJzaGOTbly5WJmH7z58KTHny31NQufEun3ktnLeX01ifs81xqNpp6vLCHFwHSCMI+5Q8duNpWR1WVizHhlpjT9Xuvgxr1pcQXdvqlwmqOYY4AJHZ55nChMuMOCBwW4b5WPDVJYRa7aX2p6na3F9Lby6rYhIJrEI91HJDaRSzSZUMNq7jhBGFZH3ZHyrl2GtePpxLFKJILx3t/MSSxllS3c3MKyBD9niQxBGl3fvpSQoKvgFz1gbxEmgeJbS3lnuNStPMi0u6njjR7hjbo6MflWM4ldkyFC/Jg5IJNPUdZ8VzkNpGlSAs81xarPEEWWAW0gRZtzAo5uPKO35G2OvQrKEz4NW1r7RqEdxqfiBNJCWwtL06AWupCTOZMIsRwdyxgs8QAUABcusrbg1TVLrwJDf20F9FdtsVxPb/wClRxeaFkkMZRR5wj3PtVGUsMKJFxu5/Th4g1PWGl1KC+Ngks1nHqDwta3a20i2ZTaiA/6xg+98IUwxzAV2DQlsfGMOmNJZR2K3w8u1RlbbM8Uchj3SSyebvUo8k65+aNgFIm3ODsW/2yWDUrnWL77Npkf2mNR5hiYR72zK8hSNk2qMLt4CgsXk3Ky5/gu3uNKvNT0zWZdK/tbzd1stmQhewUKISItxMcaFnQKMKCCfmLF26TVp7S10a+uL+WSGzit5HnkjZ1ZIwpLEFPmBAycrz6c1wcGs26aP4hgvrux07xLPFuu5LgHSpFiZjEjmcowbYSwR08xVJRN7/wCsfqNf1LRo9Mnu7rVp4IYYpfNexmdnESSIs3yR5PykBGcDdHubDISTUkWq6kfEF5psltaAK6yWpklaNpbfEIZx8rByrNMCMgjEQIAkD1JpOo3E+hxa3rCf2X5tos01pLKCloBuYlmZEYNtI3buF24HQlrE+sWawWskF5Yv9q8t4DJchFljZ0UshGd3+sXGOCzIMjcDUj6tpsaO76haKiJK7MZlAVYm2yk88BGOGP8ACeDiqdh4m0e8gtv+JvpTXE0SSeXb3ySqcuI/kbgsvmHYGwMtxgHitSeeG1t5bi4ljhgiQvJJIwVUUDJJJ4AA5zXLy+JBL4Ygv01m0BureHa1nHHLKG8xY52jUSurkGRVCr5mHwP3xZUOpqd1qIs7mTTref8AtJLS6NrZzIphmkUgRl3HC5ONo3qSrtkZU7OPt9T8Yy69FbafLfXGmtLAkV1qumeRlGiuzK8gVFJ2ukOBtizhF4D+Y/SeJLttP0iG0l1GRIGt5zqF6HVLqK2SB99xGEA+cSGHlVIBf7o4xuQ3iSfZ45R9nupojKLWV18xQNu7hSQdpZQSCRkjnkVJPPDa28txcSxwwRIXkkkYKqKBkkk8AAc5rl7zWdXmsLUWSSEi3zqU9vaiRrdt4R/KAd1aVSsoMQ8zbjJLELHN0GrXTWOjX12jxo8FvJKrSbdoKqTk7nQY47so9WHWuD07xXrGp6S8Mx1W0uobtHvnTSHS5t7WSN2jaOJkffmVPLOEfaAwy+3z20PDl/4p1fVCly89tpkcSMJ7mzMU02y5uUA2sgAaSNIGckDaD8qL5gZOktfEWlakitpV/aamWfaFs7mOQ4DIHb72MIJELd8EYBJAOhDPDcoXgljlQOyFkYMAysVYcdwwII7EEVJRUZnhW4S3aWMTujOkZYbmVSAxA6kAsoJ7bh61JRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRX//2Q==",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAANRElEQVR4Ae1aAXIcOQi0U/n/l310N43QzK6zvjikKqWJT0LQNAhmpXUu7x9vX3/e7fT+gT+/JAjIhspFqZuxiZ/SEtfBm/zmnO7p/fiU9hWjuV/BPsE0iiY+ATf1U3AzNJGefcPv72/4w5+Q8WCEBsuUIVglSKxpo17gVECd+IYxTozgohPBTURULR9wJu0KzUhOlFQZ0dwke3v/KSDGeCkUUy9ge0cgtiXe4k3FNQaR0aqXCbA3vvUNk5+Cpd7Y6ZL5KCsEu3LapdEWOPlhuvK8f7QNh/EjEAyYqSNXruG4P0ZA69cG7u3ZVk8wqYbnVWy6RVqcmdYT2nRgQsmTnvuGoSSFeVakq/QIUdkI/Ajit+ABnSsbbhJLaODivDTiEjo9pBVPerYNRyVYkQCrMvAir2qkkVTvHwtBBZHlLlWD1F4yRK6tjh2CQG4lSmhRFcTAtuPAMBjHVhJpF89bpL1iKtxt/CWge3wJ3B3n5H5KK2qeyKgLCl8FS6umbSSUGoLXslClCqHksobwUNkAaX/m3pBPxaRor7SgaNK3N6pXrctPs/sDhoz7g5diu/Gkxz3AZ92weeW5G2FAvXW7pRgexMsz70HepQGoOWUoVHX55JsFXxikJCGX1meIjEuGgiubJCJBaYAJzve3n/i0q6PV29bgEHkxITPKDVyOzQQMUsNjX9FzlQeNLYLKJ8YC2jnnPDpzsrPQCBRPKUXCZShBmrQ5xxePPOIQD9Zlh5x9bsefOGjT2ViJQhePzksIXHroK8sLWlV6AC+UbHK+OJjR7hkdrhfTT24zlLGRNF0Q5Fj86zaALh1RBbsVdJVmpbFLC7r0bkvTLHFJjrY0DyXl100/I0DumZW8I2IrvuboqB1Ll3sORF7MqWY6kNuroairCsUqIUbC4dJ2Y1TqbAKaJZabuKXsGqEMpsUMcvmmkaSvML+C+aacTHO/h225zShXPW3RxDSrwbGQyQduOZegVuoVK+V3Cfe8QvP+hQ23F+2LOf1/zy8G+gUcefxgF7B1Xl9qCpdhgArtqZFmAsPXHpCCCViQ0TkVTEBaWReyueO4DLN8MOJZscHIh4GJqmh2SkhnYVgSgRwC//ib1uXTFEtgcAuvHzhxiYn/hcbK9FgIK3gI5QITzqTgjnF7bCqDA5cCHvYCukUupsaSZ18iE/FRfwHQzzYZebYZlzPjIVI8y8OpQ9MRhAi84em73IUwSeEtkJGLDJyGIighDA9ZAKjHHQaU1FEYhegoByWfSe2xyu6IRnQKdsRhYSh3K5/MJsvdPMjvV0xbGvymhVdXKWDE7qIovS5YOh/qEaN5hLV5CNEUO58McnclkxyefJUdLGfCRHvP78YklmqgU9MOLhEvgXq8x6b/qf007KfGZwE/c+q2V68llFFPnn5WeLb5NgvAcWHrfdGbsjulMdDLIRC5KJ0EjjE0RrAZFXN7O8Pw6oZBsj2XAJttWzRgEzfIH1zcQsbvw7zyGPN20emy9DWmuuHWjEd+KOS6WwMIBXWQ4j/6lhbLQoACPARiMJMQNFhF2AXPMKDAQ9qMlyEKngJh8fswD1m+5RjwWwCOLfxgBRSPMQj7IyOR9AirHO0nDmnt2hDMg+RgkkFXBNaKSi2GyiKTY2oygLrhnESo+KzsqcErzcbCKMGIdKBXDHH2mUOq+BtA5GwYk0xwmnYjbUY3x/QxuLKB3nDPhbWQBtsxS95zNfztDfew0QUVWB1Y0JsUfo12g9tkpefO8UjXs3kBGwl0VF+6bcuutyd+H877LTYObwDjv5D1w3pwX6S47NC1MjbpSbQI9kudmOagDBqTW+BYxuZ64W3IqPty7biyh8qk6fPpBOxX8J+S/S3jdi3p/eAosbLiEk3g+RFzszfReKnKUILta05TR3S5gDqza7kl0LSZ1wVtRszbhrvnLvuFK+1NUZYlvIJZ6M+lr3Fd0H0Zvw/jrMXeWYe8w7CkLtcoWZkg4KfO0walMgtsJy4dRo7kD7eGVCLcduNbQEmIrPhIj1lSyJU0yI75YYonlpns+3scWrGqEoQkV+japQgvf+BTiCmfBpVyOy9wxsgVxhI3ZEckJsGX5NJfnJlUSyM1zKwIYFc0TPXrod1wtuLRdnL3S1EW4gRqUDhrCZcui4mBKV6Rwjcnh25AqTo/QhgohmXt+iQJ433DiVNNlntLKMWwCdShpbzixcSxRDlekckqVBhDSGCpuov3pfexWTZ0kfB/tfC3WXYmKpEVgyR5c1yECCRTg1p52YquzgXPvsA9ketypa2FNmXzYRJ0rHzxoVifJAC4D6sWySUzUv3Tw4vXUnTiUobr+o6QQ8NZ9JyMl2W5XfSX5S2h8AuIfpL50fTyhh85f1H35LPxRZZP4K8EwIbzPuTt5xrpjqVl3WHEotIJhZAlrSt1YzPOd6ep2AgaFwKRhUMAIBRGmcSqL+VLoAe5Lz86MLsMBMY4pXm4kd33LM4RfLRtyY85tV0PRsJI5CMDMFnsDzOdhd+WiYaOT+JExkUMDJMcWsJil3KEwEOKfrFIVy5T9rWEE63eB9iKj2cdqNLuOEtP4zZ0tsQVOYFPIlSQZJOTI64EaQ7i2B2ZEl7THuySuO7hIoUT8XbaLDbGfNXD0U+3pbwnxjwfRthxJDQu+/woCJr5wNPQjSY2zFptRupikIVNyrCSywPLdYkzZtmCccmQpIADegNXR0gLDaq33sfQg4MulBwrib1LzJbbxuGaeS+aZgf9H3pejfIqzml+FR9++FvLB89D7UNlOIf+ExMAtzPmFvIJwV0tTerdv0V3d7jkNnIP3/NaGW7Sy8DN60uL2LDvw7oGfTfC8kQZpqhlGDVGSJRW5Q0TtMskEqaVWkItyyup4NdpqDZZRhDg4i5HpCt0jSoHOLmX+Fc8uHXzpetyAB5dqVLGKMcYfVyIGWqQE1CmFNKNSMvCJ1t6J4EmncAJS5UDaAaf7NwIQ4s+xqwnMEDF/z2UyHOsy1TfDjpooRSN8VTmptJli0Ou6yCEN5JUFTCFFqH5rwzsvqXfkMq2FP0vAHqGJZcQLiVDeJwIicMUaXwGUM+EuOLkrQi1G/K2YWWw3Mux4SBW2lz4X8RHyVA1PiWXEOqSJcTYNs3E/PrKYkDlnBh6EWqEgnos3hIUPAoIB+wvZv04A1Ib71+shTbQzi/8e2ln0ufaRVeuj9+mHVg8TudR4EA+vZZQscv7UBR+j3YMtXnEFtSCoF5hvmtkLdJngE5C2emYcnOMxVoH8umGb7QvKir6i/hvhqHgPYXo6fYCvK9vWgHlH9TDfZIKNWKhYpL1islLz0DBdVGayq4riEv/LHCmkfdt8iAyHelVccGOr+vxCZY9M6aKLjl8xF/iqSAoBajglDLOh1UtiLZeMWmTA0CEig5OF2LSNMiTwITBW3hIySSVGZIdS2aP30lwTdAhdp5+8u6vNAw2ehZKlLb6u9ANU+BlMXYjlnklgrVdPJdKDE3tKKGCVj0vOfaKAzr2Kf6rY/97abuCscuM0Bwt3jAE7t7GPjLVjjtRl6+BudYlHqKQG0lGi2l9EcD224P/1UKfurHC2GRfa82FDCgjhTBcMLIUSrWXf3NiGCTYdU32RsQj9/ThC8vw2EvLnlvhpmMwQb075GDfy1S0/46AveE084MN97X1/+4chxZPcr7oMewv/O/v+7v5fjsj/XvppMmPs0m/kiyxGPJW/VXlFrc8cc3qx9H/0PwjDj1Q51uOieumg/j8yUTpH4MOMMxwefBpAXu7G+MzBYYP3iXroHke73ctP/TdAgVeD76mXDW2bsBQxsb0YwBr9/wcRCEUkw78jmDX515G/P4cHQaJvpGZjpcY9Utzl6RRAdAlIyAs51KnGet8q6BRh9P23CsB3zAxdg8UMmvwxWqXD+7H1sFHNNCVXuXhWLpv2NdTih9oLl/giJkfuX7X883GK+6fUODHhP5qz5phCLr6lmOU5zDDlbAkwKeXXvQDwOzNKbE1fWIqzDNhpKrPgv8Nff/l4W/EH495Njxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YCnw8MFHw93Ojxe8uGAp8PDBR8Pdzo8XvLhgKfDwwUfD3c6PF7y4YD/AbDOK5Kva5fbAAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g2.to_PIL()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manul Annotations of First 200 Samples\n",
    "\n",
    "Generated by 0.1M binary model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc:  0.04\n",
      "Wrong:  0.045\n",
      "Acc_no_cap:  0.05825242718446602\n",
      "Wrong_no_cap:  0.019417475728155338\n",
      "Non_text:  0.91\n",
      "Non_text_no_cap:  0.912621359223301\n",
      "Num_no_cap:  103\n",
      "Num_total:  200\n"
     ]
    }
   ],
   "source": [
    "capital = '1 3 5 7 8 17 19 21 23 24 26 29 30 31 33 36 38 40 41 43 48 55 57 58 59 60 61 62 65 70 72 74 76 79 80 81 82 87 88 90 93 95 96 97 99 101 104 106 110 112 115 116 117 118 120 121 122 124 126 131 132 134 138 139 141 143 145 147 148 149 150 151 154 155 157 159 160 162 164 165 166 167 172 174 176 180 181 183 186 187 189 191 192 195 197 198 199'\n",
    "wrong_word = '19 21 24 58 81 88 106 182 184'\n",
    "correct = '2 4 20 35 63 111 141 145'\n",
    "non_text = '0 1 3 5 6 7 8 9 10 11 12 13 14 15 16 17 18 22 23 25 26 27 28 29 30 31 32 33 34 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 59 60 61 62 64 65 66 67 68 69 70 72 71 73 74 75 76 77 78 79 80 82 83 84 85 86 87 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 107 108 109 110 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 142 143 144 146 147 148 149 150 151 152 153 154 155 156 157 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 183 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199'\n",
    "\n",
    "capitals = set(capital.split(' '))\n",
    "wrong_words = set(wrong_word.split(' '))\n",
    "corrects = set(correct.split(' '))\n",
    "non_texts = set(non_text.split(' '))\n",
    "\n",
    "print('Acc: ', len(corrects) / 200)\n",
    "print('Wrong: ', len(wrong_words) / 200)\n",
    "print('Acc_no_cap: ', len(corrects - capitals) / (200 - len(capitals)))\n",
    "print('Wrong_no_cap: ', len(wrong_words - capitals) / (200 - len(capitals)))\n",
    "print('Non_text: ', len(non_texts) / 200)\n",
    "print('Non_text_no_cap: ', len(non_texts - capitals) / (200 - len(capitals)))\n",
    "print('Num_no_cap: ', (200 - len(capitals)))\n",
    "print('Num_total: ', 200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluations\n",
    "\n",
    "### 1. 1M binary model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:47<00:00, 22.60it/s]\n",
      "100%|██████████| 5153/5153 [20:53<00:00,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.05200853871531147, 'tesseract': 0.050650106733941394, 'union': 0.05705414321754318, 'readability': 0.5476421502037648}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MModels/dllama_2_b/backbone/', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_generated(generated, 'lambada_stage1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "with stage 2 training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:51<00:00, 22.31it/s]\n",
      "100%|██████████| 5153/5153 [20:41<00:00,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.12458761886279837, 'tesseract': 0.13137977876964874, 'union': 0.13758975354162623, 'readability': 0.8218513487288958}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path=pixar113_stage2_path, num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated2 = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated2.append((prompt, target, gen))\n",
    "result2 = evaluate_babi(generated2, vocab)\n",
    "print(result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_generated(generated2, 'lambada_stage2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 0.1M binary model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:48<00:00,  4.15it/s]\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/100kModels/dllama_2_100k/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.035, 'tesseract': 0.025, 'union': 0.035, 'readability': 0.13}\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 1M binary model with 200 steps adversarial training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 164/5153 [00:07<03:46, 21.98it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 13\u001b[0m\n\u001b[1;32m     11\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     12\u001b[0m target \u001b[39m=\u001b[39m words[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[0;32m---> 13\u001b[0m gen \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mautoregressive_generate(prompt, \u001b[39mNone\u001b[39;49;00m, \u001b[39m4\u001b[39;49m)\n\u001b[1;32m     14\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     15\u001b[0m gen\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:229\u001b[0m, in \u001b[0;36mLatentModel.autoregressive_generate\u001b[0;34m(self, prompt, gen_idx, num_new_patches, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    227\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mautoregressive_generate\u001b[39m(\u001b[39mself\u001b[39m, prompt: TGraph, gen_idx: \u001b[39mint\u001b[39m, num_new_patches: \u001b[39mint\u001b[39m, binary_method: \u001b[39mstr\u001b[39m\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mthreshold\u001b[39m\u001b[39m'\u001b[39m, threshold: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m0.5\u001b[39m, temperature: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m1.0\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m TGraph:\n\u001b[1;32m    228\u001b[0m     \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(num_new_patches):\n\u001b[0;32m--> 229\u001b[0m         gen \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate(prompt, binary_method, threshold, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    230\u001b[0m         prompt \u001b[39m=\u001b[39m gen\n\u001b[1;32m    231\u001b[0m     \u001b[39mreturn\u001b[39;00m gen\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:215\u001b[0m, in \u001b[0;36mLatentModel._generate\u001b[0;34m(self, prompt, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    213\u001b[0m gen \u001b[39m=\u001b[39m TGraph\u001b[39m.\u001b[39mfrom_tgraph(prompt)\n\u001b[1;32m    214\u001b[0m gen\u001b[39m.\u001b[39m_value \u001b[39m=\u001b[39m prompt\u001b[39m.\u001b[39mvalue\n\u001b[0;32m--> 215\u001b[0m output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(prompt, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    216\u001b[0m \u001b[39mfor\u001b[39;00m idx, num_text \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(prompt\u001b[39m.\u001b[39mnum_text_patches):\n\u001b[1;32m    217\u001b[0m     bidx \u001b[39m=\u001b[39m (num_text \u001b[39m-\u001b[39m \u001b[39m2\u001b[39m) \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_len \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_size            \n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:97\u001b[0m, in \u001b[0;36mLatentModel.forward\u001b[0;34m(self, img, temperature)\u001b[0m\n\u001b[1;32m     95\u001b[0m     img\u001b[39m.\u001b[39mbinary()\n\u001b[1;32m     96\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcompressor \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 97\u001b[0m     recon \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlatent_forward(img)\n\u001b[1;32m     98\u001b[0m     recon\u001b[39m.\u001b[39m_labels \u001b[39m=\u001b[39m img\u001b[39m.\u001b[39m_labels\n\u001b[1;32m     99\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbinary \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_labels \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:345\u001b[0m, in \u001b[0;36mLatentLlama.latent_forward\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    344\u001b[0m     img_values \u001b[39m=\u001b[39m target\n\u001b[0;32m--> 345\u001b[0m output: BaseModelOutputWithPastAndCrossAttentions \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbackbone\u001b[39m.\u001b[39;49mforward(\n\u001b[1;32m    346\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mimg\u001b[39m.\u001b[39;49mattention_mask,\n\u001b[1;32m    347\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49mimg_values\n\u001b[1;32m    348\u001b[0m )\n\u001b[1;32m    349\u001b[0m pred \u001b[39m=\u001b[39m output\u001b[39m.\u001b[39mlast_hidden_state\n\u001b[1;32m    351\u001b[0m \u001b[39m# attention_mask = mask2img(img.attention_mask, self.latent_patch_size, self.patch_len)\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \u001b[39m# patch_width = self.latent_patch_size * self.patch_len\u001b[39;00m\n\u001b[1;32m    353\u001b[0m \u001b[39m# attention_mask.unsqueeze_(1)    # add the channel dimension\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[39m# loss = (pred[..., :-patch_width] - img_values[..., patch_width:]) ** 2\u001b[39;00m\n\u001b[1;32m    357\u001b[0m \u001b[39m# loss = (loss * attention_mask).sum() / (attention_mask.sum() * self.num_latent_channel)  # mean loss on removed patches\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:270\u001b[0m, in \u001b[0;36mLlamaForPatchCausalInference.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions, output_hidden_states, return_dict, ignore_out_proj)\u001b[0m\n\u001b[1;32m    266\u001b[0m     layer_outputs \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mcheckpoint\u001b[39m.\u001b[39mcheckpoint(\n\u001b[1;32m    267\u001b[0m         create_custom_forward(decoder_layer), hidden_states, attention_mask, position_ids\n\u001b[1;32m    268\u001b[0m     )\n\u001b[1;32m    269\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 270\u001b[0m     layer_outputs \u001b[39m=\u001b[39m decoder_layer(\n\u001b[1;32m    271\u001b[0m         hidden_states,\n\u001b[1;32m    272\u001b[0m         attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m    273\u001b[0m         position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m    274\u001b[0m         past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m    275\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m    276\u001b[0m         use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m    277\u001b[0m         padding_mask\u001b[39m=\u001b[39;49mpadding_mask,\n\u001b[1;32m    278\u001b[0m     )\n\u001b[1;32m    280\u001b[0m hidden_states \u001b[39m=\u001b[39m layer_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m    282\u001b[0m \u001b[39mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:78\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, padding_mask)\u001b[0m\n\u001b[1;32m     75\u001b[0m hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_layernorm(hidden_states)\n\u001b[1;32m     77\u001b[0m \u001b[39m# Self Attention\u001b[39;00m\n\u001b[0;32m---> 78\u001b[0m hidden_states, self_attn_weights, present_key_value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mself_attn(\n\u001b[1;32m     79\u001b[0m     hidden_states\u001b[39m=\u001b[39;49mhidden_states,\n\u001b[1;32m     80\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m     81\u001b[0m     position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m     82\u001b[0m     past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m     83\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m     84\u001b[0m     use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m     85\u001b[0m     padding_mask\u001b[39m=\u001b[39;49mpadding_mask,\n\u001b[1;32m     86\u001b[0m )\n\u001b[1;32m     87\u001b[0m hidden_states \u001b[39m=\u001b[39m residual \u001b[39m+\u001b[39m hidden_states\n\u001b[1;32m     89\u001b[0m \u001b[39m# Fully Connected\u001b[39;00m\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:361\u001b[0m, in \u001b[0;36mLlamaAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, padding_mask)\u001b[0m\n\u001b[1;32m    359\u001b[0m     kv_seq_len \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m past_key_value[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mshape[\u001b[39m-\u001b[39m\u001b[39m2\u001b[39m]\n\u001b[1;32m    360\u001b[0m cos, sin \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrotary_emb(value_states, seq_len\u001b[39m=\u001b[39mkv_seq_len)\n\u001b[0;32m--> 361\u001b[0m query_states, key_states \u001b[39m=\u001b[39m apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)\n\u001b[1;32m    363\u001b[0m \u001b[39mif\u001b[39;00m past_key_value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    364\u001b[0m     \u001b[39m# reuse k, v, self_attention\u001b[39;00m\n\u001b[1;32m    365\u001b[0m     key_states \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcat([past_key_value[\u001b[39m0\u001b[39m], key_states], dim\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:211\u001b[0m, in \u001b[0;36mapply_rotary_pos_emb\u001b[0;34m(q, k, cos, sin, position_ids)\u001b[0m\n\u001b[1;32m    209\u001b[0m cos \u001b[39m=\u001b[39m cos\u001b[39m.\u001b[39msqueeze(\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39msqueeze(\u001b[39m0\u001b[39m)  \u001b[39m# [seq_len, dim]\u001b[39;00m\n\u001b[1;32m    210\u001b[0m sin \u001b[39m=\u001b[39m sin\u001b[39m.\u001b[39msqueeze(\u001b[39m1\u001b[39m)\u001b[39m.\u001b[39msqueeze(\u001b[39m0\u001b[39m)  \u001b[39m# [seq_len, dim]\u001b[39;00m\n\u001b[0;32m--> 211\u001b[0m cos \u001b[39m=\u001b[39m cos[position_ids]\u001b[39m.\u001b[39;49munsqueeze(\u001b[39m1\u001b[39;49m)  \u001b[39m# [bs, 1, seq_len, dim]\u001b[39;00m\n\u001b[1;32m    212\u001b[0m sin \u001b[39m=\u001b[39m sin[position_ids]\u001b[39m.\u001b[39munsqueeze(\u001b[39m1\u001b[39m)  \u001b[39m# [bs, 1, seq_len, dim]\u001b[39;00m\n\u001b[1;32m    213\u001b[0m q_embed \u001b[39m=\u001b[39m (q \u001b[39m*\u001b[39m cos) \u001b[39m+\u001b[39m (rotate_half(q) \u001b[39m*\u001b[39m sin)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MGan/gan_ratio_10/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. 1M binary model with 400 steps adversarial training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:50<00:00, 22.31it/s]\n",
      "100%|██████████| 5153/5153 [20:15<00:00,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.08267028915195032, 'tesseract': 0.08849214050067922, 'union': 0.09159712788666796, 'readability': 0.8024451775664662}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MGan/gan_ratio_10/400/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 1M binary model with 600 steps adversarial training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:51<00:00, 22.29it/s]\n",
      "100%|██████████| 5153/5153 [20:23<00:00,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.060741315738404814, 'tesseract': 0.06578692024063652, 'union': 0.06753347564525519, 'readability': 0.8177760527847856}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MGan/gan_ratio_10/600/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. 1M binary model with 3000 steps adversarial training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 658/5153 [00:29<03:20, 22.44it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[12], line 13\u001b[0m\n\u001b[1;32m     11\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     12\u001b[0m target \u001b[39m=\u001b[39m words[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[0;32m---> 13\u001b[0m gen \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mautoregressive_generate(prompt, \u001b[39mNone\u001b[39;49;00m, \u001b[39m4\u001b[39;49m)\n\u001b[1;32m     14\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     15\u001b[0m gen\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:229\u001b[0m, in \u001b[0;36mLatentModel.autoregressive_generate\u001b[0;34m(self, prompt, gen_idx, num_new_patches, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    227\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mautoregressive_generate\u001b[39m(\u001b[39mself\u001b[39m, prompt: TGraph, gen_idx: \u001b[39mint\u001b[39m, num_new_patches: \u001b[39mint\u001b[39m, binary_method: \u001b[39mstr\u001b[39m\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mthreshold\u001b[39m\u001b[39m'\u001b[39m, threshold: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m0.5\u001b[39m, temperature: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m1.0\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m TGraph:\n\u001b[1;32m    228\u001b[0m     \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(num_new_patches):\n\u001b[0;32m--> 229\u001b[0m         gen \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate(prompt, binary_method, threshold, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    230\u001b[0m         prompt \u001b[39m=\u001b[39m gen\n\u001b[1;32m    231\u001b[0m     \u001b[39mreturn\u001b[39;00m gen\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:215\u001b[0m, in \u001b[0;36mLatentModel._generate\u001b[0;34m(self, prompt, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    213\u001b[0m gen \u001b[39m=\u001b[39m TGraph\u001b[39m.\u001b[39mfrom_tgraph(prompt)\n\u001b[1;32m    214\u001b[0m gen\u001b[39m.\u001b[39m_value \u001b[39m=\u001b[39m prompt\u001b[39m.\u001b[39mvalue\n\u001b[0;32m--> 215\u001b[0m output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(prompt, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    216\u001b[0m \u001b[39mfor\u001b[39;00m idx, num_text \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(prompt\u001b[39m.\u001b[39mnum_text_patches):\n\u001b[1;32m    217\u001b[0m     bidx \u001b[39m=\u001b[39m (num_text \u001b[39m-\u001b[39m \u001b[39m2\u001b[39m) \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_len \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_size            \n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:97\u001b[0m, in \u001b[0;36mLatentModel.forward\u001b[0;34m(self, img, temperature)\u001b[0m\n\u001b[1;32m     95\u001b[0m     img\u001b[39m.\u001b[39mbinary()\n\u001b[1;32m     96\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcompressor \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 97\u001b[0m     recon \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlatent_forward(img)\n\u001b[1;32m     98\u001b[0m     recon\u001b[39m.\u001b[39m_labels \u001b[39m=\u001b[39m img\u001b[39m.\u001b[39m_labels\n\u001b[1;32m     99\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbinary \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_labels \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:345\u001b[0m, in \u001b[0;36mLatentLlama.latent_forward\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    344\u001b[0m     img_values \u001b[39m=\u001b[39m target\n\u001b[0;32m--> 345\u001b[0m output: BaseModelOutputWithPastAndCrossAttentions \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbackbone\u001b[39m.\u001b[39;49mforward(\n\u001b[1;32m    346\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mimg\u001b[39m.\u001b[39;49mattention_mask,\n\u001b[1;32m    347\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49mimg_values\n\u001b[1;32m    348\u001b[0m )\n\u001b[1;32m    349\u001b[0m pred \u001b[39m=\u001b[39m output\u001b[39m.\u001b[39mlast_hidden_state\n\u001b[1;32m    351\u001b[0m \u001b[39m# attention_mask = mask2img(img.attention_mask, self.latent_patch_size, self.patch_len)\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \u001b[39m# patch_width = self.latent_patch_size * self.patch_len\u001b[39;00m\n\u001b[1;32m    353\u001b[0m \u001b[39m# attention_mask.unsqueeze_(1)    # add the channel dimension\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[39m# loss = (pred[..., :-patch_width] - img_values[..., patch_width:]) ** 2\u001b[39;00m\n\u001b[1;32m    357\u001b[0m \u001b[39m# loss = (loss * attention_mask).sum() / (attention_mask.sum() * self.num_latent_channel)  # mean loss on removed patches\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:270\u001b[0m, in \u001b[0;36mLlamaForPatchCausalInference.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions, output_hidden_states, return_dict, ignore_out_proj)\u001b[0m\n\u001b[1;32m    266\u001b[0m     layer_outputs \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mcheckpoint\u001b[39m.\u001b[39mcheckpoint(\n\u001b[1;32m    267\u001b[0m         create_custom_forward(decoder_layer), hidden_states, attention_mask, position_ids\n\u001b[1;32m    268\u001b[0m     )\n\u001b[1;32m    269\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 270\u001b[0m     layer_outputs \u001b[39m=\u001b[39m decoder_layer(\n\u001b[1;32m    271\u001b[0m         hidden_states,\n\u001b[1;32m    272\u001b[0m         attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m    273\u001b[0m         position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m    274\u001b[0m         past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m    275\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m    276\u001b[0m         use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m    277\u001b[0m         padding_mask\u001b[39m=\u001b[39;49mpadding_mask,\n\u001b[1;32m    278\u001b[0m     )\n\u001b[1;32m    280\u001b[0m hidden_states \u001b[39m=\u001b[39m layer_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m    282\u001b[0m \u001b[39mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:92\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, padding_mask)\u001b[0m\n\u001b[1;32m     90\u001b[0m residual \u001b[39m=\u001b[39m hidden_states\n\u001b[1;32m     91\u001b[0m hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpost_attention_layernorm(hidden_states)\n\u001b[0;32m---> 92\u001b[0m hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmlp(hidden_states)\n\u001b[1;32m     93\u001b[0m hidden_states \u001b[39m=\u001b[39m residual \u001b[39m+\u001b[39m hidden_states\n\u001b[1;32m     95\u001b[0m outputs \u001b[39m=\u001b[39m (hidden_states,)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:247\u001b[0m, in \u001b[0;36mLlamaMLP.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    245\u001b[0m     down_proj \u001b[39m=\u001b[39m \u001b[39msum\u001b[39m(down_proj)\n\u001b[1;32m    246\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 247\u001b[0m     down_proj \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdown_proj(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mact_fn(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgate_proj(x)) \u001b[39m*\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mup_proj(x))\n\u001b[1;32m    249\u001b[0m \u001b[39mreturn\u001b[39;00m down_proj\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mlinear(\u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MGan/gan_ratio_10/3000/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "for txt in tqdm(data['text']):\n",
    "    words = txt.split()\n",
    "    prompt = process_lambda(' '.join(words[:-1]))\n",
    "    prompt = TGraph.from_text([prompt])\n",
    "    prompt = prompt._spacing_text(3)\n",
    "    prompt.set_device('cuda')\n",
    "    target = words[-1]\n",
    "    gen = model.autoregressive_generate(prompt, None, 4)\n",
    "    prompt.set_device('cpu')\n",
    "    gen.set_device('cpu')\n",
    "    generated.append((prompt, target, gen))\n",
    "result = evaluate_babi(generated, vocab)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GAN Model with 7.0 Gan_ratio\n",
    "\n",
    "```\n",
    "/exports/eddie3_homes_local/s1891075/msc_project/train.py --task gan_pretrain --model LatentLlama --exp_type full_gan --backbone_path /home/s1891075/msc_project/storage/checkpoints/pretrain/lpixel_pretrain/LatentLlama/20231202-215356/900000/backbone --dataset_path storage/BooksAndWiki2 --discriminator_path self --shuffle_dataset false --optim AdamW --lr 3e-5 --beta1 0.9 --beta2 0.95 --decay 0.1 --total_steps 5000 --stop_step 5000 --warm_up_step 100 --save_freq 200 --eval_freq 200 --gan_lr 3e-5 --gan_lr_warm_up_steps 100 --gan_total_steps 5000 --gan_ratio 7.0 --gan_ratio_warm_up_steps 100 --seed 42 --batch_size 128 --sub_size 64 --dpi 80 --font_size 8 --font_file PixeloidSans-mLxMm.ttf --pixels_per_patch 8 --patch_len 2 --num_channel 1 --binary true --rgb false --max_seq_length 720 --mix_precision fp16 --half_coder false --mp_workers 16 --prerendered false --is_continue_train false --num_gpu_per_node 2 --num_node 1\n",
    "```\n",
    "\n",
    "#### 1. 200 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:49<00:00, 22.44it/s]\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_7_0/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702255616 (unix time) try \"date -d @1702255616\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1861542 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702255616 (unix time) try \"date -d @1702255616\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1861562 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702255616 (unix time) try \"date -d @1702255616\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1861556 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.03473704638074908, 0.03764797205511353, 0.7259848631864934)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 400 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:16<00:00, 26.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702255982 (unix time) try \"date -d @1702255982\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1884260 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.03240830584125752, 0.03784203376673782, 0.6627207451969727)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_7_0/400/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. 600 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:17<00:00, 26.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702256343 (unix time) try \"date -d @1702256343\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1907462 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702256343 (unix time) try \"date -d @1702256343\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1907456 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.03279642926450611, 0.03978265088298079, 0.6708713370851931)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_7_0/600/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GAN Model with 5.0 Gan_ratio\n",
    "\n",
    "```\n",
    "/exports/eddie3_homes_local/s1891075/msc_project/train.py --task gan_pretrain --model LatentLlama --exp_type full_gan --backbone_path /home/s1891075/msc_project/storage/checkpoints/pretrain/lpixel_pretrain/LatentLlama/20231202-215356/900000/backbone --dataset_path storage/BooksAndWiki2 --discriminator_path self --shuffle_dataset false --optim AdamW --lr 3e-5 --beta1 0.9 --beta2 0.95 --decay 0.1 --total_steps 5000 --stop_step 5000 --warm_up_step 100 --save_freq 200 --eval_freq 200 --gan_lr 3e-5 --gan_lr_warm_up_steps 100 --gan_total_steps 5000 --gan_ratio 5.0 --gan_ratio_warm_up_steps 100 --seed 42 --batch_size 128 --sub_size 64 --dpi 80 --font_size 8 --font_file PixeloidSans-mLxMm.ttf --pixels_per_patch 8 --patch_len 2 --num_channel 1 --binary true --rgb false --max_seq_length 720 --mix_precision fp16 --half_coder false --mp_workers 16 --prerendered false --is_continue_train false --num_gpu_per_node 2 --num_node 1\n",
    "```\n",
    "\n",
    "#### 1. 200 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:17<00:00, 26.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702256707 (unix time) try \"date -d @1702256707\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1930383 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702256707 (unix time) try \"date -d @1702256707\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1930397 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702256707 (unix time) try \"date -d @1702256707\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1930387 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.021346788278672618, 0.022705220260042694, 0.7292839122841064)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_5_0/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 400 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:16<00:00, 26.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702257067 (unix time) try \"date -d @1702257067\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1953435 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.0027168639627401514, 0.002522802251115855, 0.7374345041723268)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_5_0/400/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. 600 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [03:16<00:00, 26.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1702257425 (unix time) try \"date -d @1702257425\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e8001c3433) received by PID 1976549 (TID 0x7f5206588740) from PID 1848371 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.004657481078983117, 0.00504560450223171, 0.8847273432951679)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_5_0/600/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GAN Model with 3.0 Gan_ratio\n",
    "\n",
    "```\n",
    "/exports/eddie3_homes_local/s1891075/msc_project/train.py --task gan_pretrain --model LatentLlama --exp_type full_gan --backbone_path /home/s1891075/msc_project/storage/checkpoints/pretrain/lpixel_pretrain/LatentLlama/20231202-215356/900000/backbone --dataset_path storage/BooksAndWiki2 --discriminator_path self --shuffle_dataset false --optim AdamW --lr 3e-5 --beta1 0.9 --beta2 0.95 --decay 0.1 --total_steps 5000 --stop_step 5000 --warm_up_step 100 --save_freq 200 --eval_freq 200 --gan_lr 3e-5 --gan_lr_warm_up_steps 100 --gan_total_steps 5000 --gan_ratio 3.0 --gan_ratio_warm_up_steps 100 --seed 42 --batch_size 128 --sub_size 64 --dpi 80 --font_size 8 --font_file PixeloidSans-mLxMm.ttf --pixels_per_patch 8 --patch_len 2 --num_channel 1 --binary true --rgb false --max_seq_length 720 --mix_precision fp16 --half_coder false --mp_workers 16 --prerendered false --is_continue_train false --num_gpu_per_node 2 --num_node 1\n",
    "```\n",
    "\n",
    "#### 1. 200 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "Can't load the configuration of '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone' is the correct path to a directory containing a config.json file",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mHFValidationError\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/configuration_utils.py:675\u001b[0m, in \u001b[0;36mPretrainedConfig._get_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    673\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    674\u001b[0m     \u001b[39m# Load from local folder or from cache or download from model Hub and cache\u001b[39;00m\n\u001b[0;32m--> 675\u001b[0m     resolved_config_file \u001b[39m=\u001b[39m cached_file(\n\u001b[1;32m    676\u001b[0m         pretrained_model_name_or_path,\n\u001b[1;32m    677\u001b[0m         configuration_file,\n\u001b[1;32m    678\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m    679\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[1;32m    680\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[1;32m    681\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[1;32m    682\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[1;32m    683\u001b[0m         token\u001b[39m=\u001b[39;49mtoken,\n\u001b[1;32m    684\u001b[0m         user_agent\u001b[39m=\u001b[39;49muser_agent,\n\u001b[1;32m    685\u001b[0m         revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[1;32m    686\u001b[0m         subfolder\u001b[39m=\u001b[39;49msubfolder,\n\u001b[1;32m    687\u001b[0m         _commit_hash\u001b[39m=\u001b[39;49mcommit_hash,\n\u001b[1;32m    688\u001b[0m     )\n\u001b[1;32m    689\u001b[0m     commit_hash \u001b[39m=\u001b[39m extract_commit_hash(resolved_config_file, commit_hash)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/utils/hub.py:429\u001b[0m, in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    428\u001b[0m     \u001b[39m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[0;32m--> 429\u001b[0m     resolved_file \u001b[39m=\u001b[39m hf_hub_download(\n\u001b[1;32m    430\u001b[0m         path_or_repo_id,\n\u001b[1;32m    431\u001b[0m         filename,\n\u001b[1;32m    432\u001b[0m         subfolder\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m \u001b[39mif\u001b[39;49;00m \u001b[39mlen\u001b[39;49m(subfolder) \u001b[39m==\u001b[39;49m \u001b[39m0\u001b[39;49m \u001b[39melse\u001b[39;49;00m subfolder,\n\u001b[1;32m    433\u001b[0m         repo_type\u001b[39m=\u001b[39;49mrepo_type,\n\u001b[1;32m    434\u001b[0m         revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[1;32m    435\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m    436\u001b[0m         user_agent\u001b[39m=\u001b[39;49muser_agent,\n\u001b[1;32m    437\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[1;32m    438\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[1;32m    439\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[1;32m    440\u001b[0m         token\u001b[39m=\u001b[39;49mtoken,\n\u001b[1;32m    441\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[1;32m    442\u001b[0m     )\n\u001b[1;32m    443\u001b[0m \u001b[39mexcept\u001b[39;00m GatedRepoError \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:110\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.<locals>._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[39mif\u001b[39;00m arg_name \u001b[39min\u001b[39;00m [\u001b[39m\"\u001b[39m\u001b[39mrepo_id\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mfrom_id\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mto_id\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[0;32m--> 110\u001b[0m     validate_repo_id(arg_value)\n\u001b[1;32m    112\u001b[0m \u001b[39melif\u001b[39;00m arg_name \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mtoken\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mand\u001b[39;00m arg_value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:158\u001b[0m, in \u001b[0;36mvalidate_repo_id\u001b[0;34m(repo_id)\u001b[0m\n\u001b[1;32m    157\u001b[0m \u001b[39mif\u001b[39;00m repo_id\u001b[39m.\u001b[39mcount(\u001b[39m\"\u001b[39m\u001b[39m/\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m--> 158\u001b[0m     \u001b[39mraise\u001b[39;00m HFValidationError(\n\u001b[1;32m    159\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mRepo id must be in the form \u001b[39m\u001b[39m'\u001b[39m\u001b[39mrepo_name\u001b[39m\u001b[39m'\u001b[39m\u001b[39m or \u001b[39m\u001b[39m'\u001b[39m\u001b[39mnamespace/repo_name\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    160\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mrepo_id\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m. Use `repo_type` argument if needed.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    161\u001b[0m     )\n\u001b[1;32m    163\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m REPO_ID_REGEX\u001b[39m.\u001b[39mmatch(repo_id):\n",
      "\u001b[0;31mHFValidationError\u001b[0m: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone'. Use `repo_type` argument if needed.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[29], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model \u001b[39m=\u001b[39m LatentLlama(backbone_path\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone\u001b[39;49m\u001b[39m'\u001b[39;49m, num_channels\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m, patch_len\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, patch_size\u001b[39m=\u001b[39;49m\u001b[39m8\u001b[39;49m)\n\u001b[1;32m      3\u001b[0m model\u001b[39m.\u001b[39meval()\n\u001b[1;32m      4\u001b[0m model\u001b[39m.\u001b[39mcuda()\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:56\u001b[0m, in \u001b[0;36mLatentModel.__init__\u001b[0;34m(self, compressor_path, backbone_path, compressor_name, num_channels, num_labels, patch_size, patch_len, binary)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_latent_channel \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcompressor\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mnum_latent_channel\n\u001b[1;32m     54\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlatent_patch_size \u001b[39m=\u001b[39m patch_size \u001b[39m/\u001b[39m\u001b[39m/\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcompressor\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mcompress_ratio\n\u001b[0;32m---> 56\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackbone \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mload_backbone(\n\u001b[1;32m     57\u001b[0m     path\u001b[39m=\u001b[39;49mbackbone_path,\n\u001b[1;32m     58\u001b[0m     num_latent_channel\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnum_latent_channel,\n\u001b[1;32m     59\u001b[0m     latent_patch_size\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlatent_patch_size,\n\u001b[1;32m     60\u001b[0m     patch_len\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpatch_len,\n\u001b[1;32m     61\u001b[0m     num_labels\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnum_labels,\n\u001b[1;32m     62\u001b[0m     binary\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbinary\n\u001b[1;32m     63\u001b[0m )\n\u001b[1;32m     65\u001b[0m problems, ok \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_structure()\n\u001b[1;32m     66\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m ok:\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:314\u001b[0m, in \u001b[0;36mLatentLlama.load_backbone\u001b[0;34m(self, path, num_latent_channel, latent_patch_size, patch_len, num_labels, binary)\u001b[0m\n\u001b[1;32m    313\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mload_backbone\u001b[39m(\u001b[39mself\u001b[39m, path: \u001b[39mstr\u001b[39m \u001b[39m|\u001b[39m PathLike, num_latent_channel: \u001b[39mint\u001b[39m, latent_patch_size: \u001b[39mint\u001b[39m, patch_len: \u001b[39mint\u001b[39m, num_labels: \u001b[39mint\u001b[39m, binary: \u001b[39mbool\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m nn\u001b[39m.\u001b[39mModule:\n\u001b[0;32m--> 314\u001b[0m     Llama_config \u001b[39m=\u001b[39m LlamaConfig\u001b[39m.\u001b[39;49mfrom_pretrained(path)\n\u001b[1;32m    316\u001b[0m     \u001b[39msetattr\u001b[39m(Llama_config, \u001b[39m'\u001b[39m\u001b[39mnum_channel\u001b[39m\u001b[39m'\u001b[39m, num_latent_channel)\n\u001b[1;32m    317\u001b[0m     \u001b[39msetattr\u001b[39m(Llama_config, \u001b[39m'\u001b[39m\u001b[39mpatch_size\u001b[39m\u001b[39m'\u001b[39m, latent_patch_size)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/configuration_utils.py:591\u001b[0m, in \u001b[0;36mPretrainedConfig.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, cache_dir, force_download, local_files_only, token, revision, **kwargs)\u001b[0m\n\u001b[1;32m    587\u001b[0m kwargs[\u001b[39m\"\u001b[39m\u001b[39mrevision\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m revision\n\u001b[1;32m    589\u001b[0m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_set_token_in_kwargs(kwargs, token)\n\u001b[0;32m--> 591\u001b[0m config_dict, kwargs \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49mget_config_dict(pretrained_model_name_or_path, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    592\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mmodel_type\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m config_dict \u001b[39mand\u001b[39;00m \u001b[39mhasattr\u001b[39m(\u001b[39mcls\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mmodel_type\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mand\u001b[39;00m config_dict[\u001b[39m\"\u001b[39m\u001b[39mmodel_type\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m!=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mmodel_type:\n\u001b[1;32m    593\u001b[0m     logger\u001b[39m.\u001b[39mwarning(\n\u001b[1;32m    594\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mYou are using a model of type \u001b[39m\u001b[39m{\u001b[39;00mconfig_dict[\u001b[39m'\u001b[39m\u001b[39mmodel_type\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m}\u001b[39;00m\u001b[39m to instantiate a model of type \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    595\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mcls\u001b[39m\u001b[39m.\u001b[39mmodel_type\u001b[39m}\u001b[39;00m\u001b[39m. This is not supported for all configurations of models and can yield errors.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    596\u001b[0m     )\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/configuration_utils.py:620\u001b[0m, in \u001b[0;36mPretrainedConfig.get_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    618\u001b[0m original_kwargs \u001b[39m=\u001b[39m copy\u001b[39m.\u001b[39mdeepcopy(kwargs)\n\u001b[1;32m    619\u001b[0m \u001b[39m# Get config dict associated with the base config file\u001b[39;00m\n\u001b[0;32m--> 620\u001b[0m config_dict, kwargs \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49m_get_config_dict(pretrained_model_name_or_path, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    621\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m config_dict:\n\u001b[1;32m    622\u001b[0m     original_kwargs[\u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m config_dict[\u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/configuration_utils.py:696\u001b[0m, in \u001b[0;36mPretrainedConfig._get_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    693\u001b[0m         \u001b[39mraise\u001b[39;00m\n\u001b[1;32m    694\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m    695\u001b[0m         \u001b[39m# For any other exception, we throw a generic error.\u001b[39;00m\n\u001b[0;32m--> 696\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mEnvironmentError\u001b[39;00m(\n\u001b[1;32m    697\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCan\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt load the configuration of \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mpretrained_model_name_or_path\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m. If you were trying to load it\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    698\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39m from \u001b[39m\u001b[39m'\u001b[39m\u001b[39mhttps://huggingface.co/models\u001b[39m\u001b[39m'\u001b[39m\u001b[39m, make sure you don\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt have a local directory with the same\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    699\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m name. Otherwise, make sure \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mpretrained_model_name_or_path\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m is the correct path to a directory\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    700\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m containing a \u001b[39m\u001b[39m{\u001b[39;00mconfiguration_file\u001b[39m}\u001b[39;00m\u001b[39m file\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    701\u001b[0m         )\n\u001b[1;32m    703\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    704\u001b[0m     \u001b[39m# Load config dict\u001b[39;00m\n\u001b[1;32m    705\u001b[0m     config_dict \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\u001b[39m.\u001b[39m_dict_from_json_file(resolved_config_file)\n",
      "\u001b[0;31mOSError\u001b[0m: Can't load the configuration of '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure '/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone' is the correct path to a directory containing a config.json file"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 400 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/400/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. 600 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_3_0/600/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GAN Model with 1.0 Gan_ratio\n",
    "\n",
    "```\n",
    "/exports/eddie3_homes_local/s1891075/msc_project/train.py --task gan_pretrain --model LatentLlama --exp_type full_gan --backbone_path /home/s1891075/msc_project/storage/checkpoints/pretrain/lpixel_pretrain/LatentLlama/20231202-215356/900000/backbone --dataset_path storage/BooksAndWiki2 --discriminator_path self --shuffle_dataset false --optim AdamW --lr 3e-5 --beta1 0.9 --beta2 0.95 --decay 0.1 --total_steps 5000 --stop_step 5000 --warm_up_step 100 --save_freq 200 --eval_freq 200 --gan_lr 3e-5 --gan_lr_warm_up_steps 100 --gan_total_steps 5000 --gan_ratio 1.0 --gan_ratio_warm_up_steps 100 --seed 42 --batch_size 128 --sub_size 64 --dpi 80 --font_size 8 --font_file PixeloidSans-mLxMm.ttf --pixels_per_patch 8 --patch_len 2 --num_channel 1 --binary true --rgb false --max_seq_length 720 --mix_precision fp16 --half_coder false --mp_workers 16 --prerendered false --is_continue_train false --num_gpu_per_node 2 --num_node 1\n",
    "```\n",
    "\n",
    "#### 1. 200 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_1_0/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 400 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_1_0/400/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. 600 steps ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGan/gan_ratio_1_0/600/backbone', num_channels=1, patch_len=2, patch_size=8)\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n",
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation for 85M models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='storage/checkpoints/1MGan/gan_ratio_10/200/backbone', num_channels=1, patch_len=2, patch_size=8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5153 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▎         | 186/5153 [00:07<03:11, 25.96it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[32], line 12\u001b[0m\n\u001b[1;32m     10\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     11\u001b[0m target \u001b[39m=\u001b[39m words[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[0;32m---> 12\u001b[0m gen \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mautoregressive_generate(prompt, \u001b[39mNone\u001b[39;49;00m, \u001b[39m4\u001b[39;49m)\n\u001b[1;32m     13\u001b[0m prompt\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m     14\u001b[0m gen\u001b[39m.\u001b[39mset_device(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:229\u001b[0m, in \u001b[0;36mLatentModel.autoregressive_generate\u001b[0;34m(self, prompt, gen_idx, num_new_patches, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    227\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mautoregressive_generate\u001b[39m(\u001b[39mself\u001b[39m, prompt: TGraph, gen_idx: \u001b[39mint\u001b[39m, num_new_patches: \u001b[39mint\u001b[39m, binary_method: \u001b[39mstr\u001b[39m\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mthreshold\u001b[39m\u001b[39m'\u001b[39m, threshold: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m0.5\u001b[39m, temperature: \u001b[39mfloat\u001b[39m\u001b[39m=\u001b[39m\u001b[39m1.0\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m TGraph:\n\u001b[1;32m    228\u001b[0m     \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(num_new_patches):\n\u001b[0;32m--> 229\u001b[0m         gen \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate(prompt, binary_method, threshold, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    230\u001b[0m         prompt \u001b[39m=\u001b[39m gen\n\u001b[1;32m    231\u001b[0m     \u001b[39mreturn\u001b[39;00m gen\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:215\u001b[0m, in \u001b[0;36mLatentModel._generate\u001b[0;34m(self, prompt, binary_method, threshold, temperature)\u001b[0m\n\u001b[1;32m    213\u001b[0m gen \u001b[39m=\u001b[39m TGraph\u001b[39m.\u001b[39mfrom_tgraph(prompt)\n\u001b[1;32m    214\u001b[0m gen\u001b[39m.\u001b[39m_value \u001b[39m=\u001b[39m prompt\u001b[39m.\u001b[39mvalue\n\u001b[0;32m--> 215\u001b[0m output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(prompt, temperature\u001b[39m=\u001b[39;49mtemperature)\n\u001b[1;32m    216\u001b[0m \u001b[39mfor\u001b[39;00m idx, num_text \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(prompt\u001b[39m.\u001b[39mnum_text_patches):\n\u001b[1;32m    217\u001b[0m     bidx \u001b[39m=\u001b[39m (num_text \u001b[39m-\u001b[39m \u001b[39m2\u001b[39m) \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_len \u001b[39m*\u001b[39m prompt\u001b[39m.\u001b[39mpatch_size            \n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/latent_model.py:97\u001b[0m, in \u001b[0;36mLatentModel.forward\u001b[0;34m(self, img, temperature)\u001b[0m\n\u001b[1;32m     95\u001b[0m     img\u001b[39m.\u001b[39mbinary()\n\u001b[1;32m     96\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcompressor \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 97\u001b[0m     recon \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlatent_forward(img)\n\u001b[1;32m     98\u001b[0m     recon\u001b[39m.\u001b[39m_labels \u001b[39m=\u001b[39m img\u001b[39m.\u001b[39m_labels\n\u001b[1;32m     99\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbinary \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_labels \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:345\u001b[0m, in \u001b[0;36mLatentLlama.latent_forward\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    344\u001b[0m     img_values \u001b[39m=\u001b[39m target\n\u001b[0;32m--> 345\u001b[0m output: BaseModelOutputWithPastAndCrossAttentions \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbackbone\u001b[39m.\u001b[39;49mforward(\n\u001b[1;32m    346\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mimg\u001b[39m.\u001b[39;49mattention_mask,\n\u001b[1;32m    347\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49mimg_values\n\u001b[1;32m    348\u001b[0m )\n\u001b[1;32m    349\u001b[0m pred \u001b[39m=\u001b[39m output\u001b[39m.\u001b[39mlast_hidden_state\n\u001b[1;32m    351\u001b[0m \u001b[39m# attention_mask = mask2img(img.attention_mask, self.latent_patch_size, self.patch_len)\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \u001b[39m# patch_width = self.latent_patch_size * self.patch_len\u001b[39;00m\n\u001b[1;32m    353\u001b[0m \u001b[39m# attention_mask.unsqueeze_(1)    # add the channel dimension\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[39m# loss = (pred[..., :-patch_width] - img_values[..., patch_width:]) ** 2\u001b[39;00m\n\u001b[1;32m    357\u001b[0m \u001b[39m# loss = (loss * attention_mask).sum() / (attention_mask.sum() * self.num_latent_channel)  # mean loss on removed patches\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:270\u001b[0m, in \u001b[0;36mLlamaForPatchCausalInference.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions, output_hidden_states, return_dict, ignore_out_proj)\u001b[0m\n\u001b[1;32m    266\u001b[0m     layer_outputs \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mcheckpoint\u001b[39m.\u001b[39mcheckpoint(\n\u001b[1;32m    267\u001b[0m         create_custom_forward(decoder_layer), hidden_states, attention_mask, position_ids\n\u001b[1;32m    268\u001b[0m     )\n\u001b[1;32m    269\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 270\u001b[0m     layer_outputs \u001b[39m=\u001b[39m decoder_layer(\n\u001b[1;32m    271\u001b[0m         hidden_states,\n\u001b[1;32m    272\u001b[0m         attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m    273\u001b[0m         position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m    274\u001b[0m         past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m    275\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m    276\u001b[0m         use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m    277\u001b[0m         padding_mask\u001b[39m=\u001b[39;49mpadding_mask,\n\u001b[1;32m    278\u001b[0m     )\n\u001b[1;32m    280\u001b[0m hidden_states \u001b[39m=\u001b[39m layer_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m    282\u001b[0m \u001b[39mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/src/projects/msc_project/LatentPixel/modeling/llama.py:92\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, padding_mask)\u001b[0m\n\u001b[1;32m     90\u001b[0m residual \u001b[39m=\u001b[39m hidden_states\n\u001b[1;32m     91\u001b[0m hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpost_attention_layernorm(hidden_states)\n\u001b[0;32m---> 92\u001b[0m hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmlp(hidden_states)\n\u001b[1;32m     93\u001b[0m hidden_states \u001b[39m=\u001b[39m residual \u001b[39m+\u001b[39m hidden_states\n\u001b[1;32m     95\u001b[0m outputs \u001b[39m=\u001b[39m (hidden_states,)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/data/miniconda3/envs/pixel/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:229\u001b[0m, in \u001b[0;36mLlamaMLP.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    226\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdown_proj \u001b[39m=\u001b[39m nn\u001b[39m.\u001b[39mLinear(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mintermediate_size, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhidden_size, bias\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n\u001b[1;32m    227\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mact_fn \u001b[39m=\u001b[39m ACT2FN[config\u001b[39m.\u001b[39mhidden_act]\n\u001b[0;32m--> 229\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x):\n\u001b[1;32m    230\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mpretraining_tp \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    231\u001b[0m         \u001b[39mslice\u001b[39m \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mintermediate_size \u001b[39m/\u001b[39m\u001b[39m/\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mpretraining_tp\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text']):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--------------------------------------\n",
      "C++ Traceback (most recent call last):\n",
      "--------------------------------------\n",
      "No stack trace in paddle, may be caused by external reasons.\n",
      "\n",
      "----------------------\n",
      "Error Message Summary:\n",
      "----------------------\n",
      "FatalError: `Termination signal` is detected by the operating system.\n",
      "  [TimeInfo: *** Aborted at 1703041933 (unix time) try \"date -d @1703041933\" if you are using GNU date ***]\n",
      "  [SignalInfo: *** SIGTERM (@0x3e80008c12e) received by PID 625468 (TID 0x7f39bfa06740) from PID 573742 ***]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.105, 0.11, 0.115, 0.7)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(generated, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coder path is none, do not load compressor for this model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:07<00:00, 28.33it/s]\n"
     ]
    }
   ],
   "source": [
    "model = LatentLlama(backbone_path='/home/tai/src/projects/msc_project/storage/checkpoints/1MGAN_85M/gan_ratio_1_4_10/ckpts/1000', num_channels=1, patch_len=2, patch_size=8)\n",
    "model.eval()\n",
    "model.cuda()\n",
    "generated = []\n",
    "with torch.no_grad():\n",
    "    for txt in tqdm(data['text'][:200]):\n",
    "        words = txt.split()\n",
    "        prompt = process_lambda(' '.join(words[:-1]))\n",
    "        prompt = TGraph.from_text([prompt])\n",
    "        prompt = prompt._spacing_text(3)\n",
    "        prompt.set_device('cuda')\n",
    "        target = words[-1]\n",
    "        gen = model.autoregressive_generate(prompt, None, 4)\n",
    "        prompt.set_device('cpu')\n",
    "        gen.set_device('cpu')\n",
    "        generated.append((prompt, target, gen))\n",
    "        \n",
    "for g in generated:\n",
    "    g[2]._value = g[2]._value.to('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:48<00:00,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.065, 'tesseract': 0.06, 'union': 0.065, 'readability': 0.385}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ans7_100 = evaluate_babi(generated, vocab)\n",
    "print(ans7_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf6x/EouDpUAtYPOk/tCyJXyRLhPtUW9sEHGF3Nu/hxuBBGRj+FJ/Ev/ElOtTz3LX+lG4uxJaLCtrOvkhUAABVmEkhYMTlkJUIAVrLubrxhDFfXUVzqUjqmp3cNubKMrut59ttAMR7ikqNk8722Aoy853LO+1mTxQ2jyyfu7OWS7muNqfvraQMLePb/AA/MZF3A5/0TJGJRWW2s6/J4z1O0t5buSC21O3gt4Es1MDxNFA1x5soBKlFdnXJTLHAMv3Fx5brxXqVzpttdw6lLAXtbi+jfTwiW92l9Zl443ABaJF8wq2CCNx8yTB2aGkXXjCO106e7udSuJPs+mTzxT2Uah3uZfLuY2CxqQIVXeACCpYlyy4AueKLvWLLVtTl08Twq9pZJHNBYPK7t5lyXUSJDKRgbD8yOo3YAUyBhoC61lPAkNwn25tQbYZnmiRp0iMo8x1RY1BZYyzIpjDHCho92Uqvot/4ll0DxHKYp7m+glcaQt9arbGdBbxtEWUEfectuJK8lgRHjYuPey6pcCxuNGudc1i+tLuSWJNX0z7KiP9iuwnPkw5VnKKSTxleVzzHpus+KpbKZbufUktFuIhNdR6fLJdwxlJclFeziVxvWBcLFIVDuSQNpXQ8LXfjTUtQaPVzJZ26IJZWktVjcsLu6HloPmXDRpDuO98KBtJMnmCx4vv8AWoNZsbbTJ9SWJ0BkSwtCWJLY/wBa9vLETxwrNCF6sxDApnkavpmseJTHdaylvcaxFJNcRWAleK3NouHgAiIkPmokTACQqq5IBy5De+NJLXU7tZrtEsdM+1WcJsF330qy3Plq+VyC8SQGSNQrguNvlcqSPxH4h0/xZey60ZLbR7d7l5gbeZ4kto1dkkRkttocqqMczsPmcbQxCr0Hii71e2uLJdPN2kDJKS9pai4Z7gFPJidTwImBk3MSgG1f3keeef0K98aS6bDqOoTXfmR3GnRGxawVPMWWK2Fw7/LuwjSTMNu3ayNuLKNqyeG5NdvfGWn3Wqx3x8vSrlbkz2Qijt7p5LYvBG4xvjGz5SQ2QGIkk52V0m8XRR6/NbGe2hsIrm7tLSPTkH2yf7XeFUzt+ZXRYSwUb23owdSxLx65feI7q68SW0H9pXUCWV2ILddPxAzCJgisk1t+8G7A+SaTe2CECFghrFx4rm121ktbS7ubu0vb2WO2kswtrERbXKWpWbK5Eishf5nAZsFojhG3PAuo63ffb01d55I4/LMLXNvLHJk7twy1rbqV4XAVSQS2TgqBn+B7TUYNcgnvxfQtP4a04GA2SxQRyJ5geMFYwEZSwITIP71uCAoSndX3iO3kgtbb+0rcPe3haKw0/wAtmU3koRt7W0kJLJtI3NF13u7B9y9R4ou9XtriyXTzdpAySkvaWouGe4BTyYnU8CJgZNzEoBtX95HnnPvNS16P4c6PdNJdw6xNb25utlk7S+YYwXG1IZfKO7OSYmHBXClgy58dx4tlEUi3msxoLexXZLawMXae5ljmkJ8lTmOLY6jam0BTInLLVfWNY8aQ6Te/ZxqS3lvb3CWHkaas32ySOadC0wK4U+UkMi4KBmkO1ZBiOtTwTYXlp4i16W5tJ4Y5d3lvJGVD/wDEw1B+CevyujfR1PQio/DsviiPQdLsJreSG4mSwmjcWyRRW1uscH2mF1CjY+5ZQF25/fJtwEfy6aat4x+zRNbLfXF832c39vdWfkw2s5uYFaKFxF88JRrgNIPO2rGrbu7bGrXHiLS5YLGO++1TarF9lt7kRRxi1utzMzqhzuURM7hWPS0ClmaTJk8cX+q2VvZjSp7uN3di4tbSSRiABjLrb3AUc/dKAtkEMNpDY86a5Y+I9SvIZdZaSfR9PB820ik8tVnkFwQ0cZRp40ZnCAtuaQ4V1CqK8Ta/f6no91qE+si3s/EeyBv7NVGe2aywkkihGKguzKzfKB5r5WMgLHYl1rxfJoloLO2u21C00zZqvnWbRD7UXgDNGxjIkKoLph5SyLkL8r5RWr2smvakfDN5qNxqQ+ya6ykxWLlzE1s4DTb7WM43sU3qioFk5O4bxTWXxTouii1spNVeP+1dRNzcPalZgxnLRbQlpLujdWeQsI9u7gOowh7yPUdVuPCE1zbW0cuuR2W4QtBJDFJcmEOFUS7W2FmA5II5BIYEDP8ACl7q9zrOqRXc2pXGlx29u1lcahYC2kkYtN5uQFUkghByiYAX5Tne/Nxa/wCLl1SygkkvizWn2q+gfT0Xy5EubUXMcJxuljSJ5ApUMW3fLJK2Ampdax4jnupbi2GpQ6ZNcSGxZNN3SuyxQCOKSJ1DJE8n2osz+X91f3iKVJj0C98c3mhXt3cpi7trSOS1t7iARvdXDWMTFHBChYxMz5wcljtyioQ2OX8SnxLPf6baX2pMsUK2VxqmmrA0pS31EgNwmzErom7bEdrqOQ299C3vvF15r0VlaX+qjSZJYES/vdJSOQxmK7MpIKrhg6QqCyIAfLyjhsy9Ab3XR4OmkKT/AG+K7e3aUQAzG3S5MZnVMYaTyAZAApDNjCEEKcPSJ/GGq629nNqGpWem77nZdyaZHHK8SpaGEjcpVXYvMTuXn94NiEAR15PEfivZpd1drd2M9xcaYktqtgPs4gmaASszvykvmvJFt3FgoB8sf60U7W+8YaqVtdYt5ywu7GeeD7NMRDIl7blhE/2aNPLVfMJ/ezHCqQxAZm9E1DStM8TabCt/bSTW7ozoknmQsBJE8bZX5WBKSuMHBGexHHD3Ol65pniHW9U0u0kvr+41hmg+1WkW0qmluYsSBVITzWEW7cOhXILOWksfEOuxeI5Ea51W80W0+zSyNPpYS5eGRLxSxiRBIf3yQrxGhwmdpXdI+podx4r1DXJku5JLbS9i3KSSW4WRkN1clI1BXgtCLffu+ZAMbQ7l1w9L1vxxIt4t9HPHN5SNcILSVzbHzYxKbcm2RG2RtMQu+4LFE27xu33LOPV28HePJntbue7uHmez+1WAV7pRZQpGxhIIYttwV2jJyCiHKLueGdIlsfEvi6+nad2vNQi8t5oUXdGtvFjayoCygsyDJIGz+8XLZa6p4oltbqFXk83SUSzvpYokke4kaVC86IAdrrbATKgDgm4CYYxlTJp+t6xZ3kVxf/2rdaEPtEUU76Y5uZmxbmNpIY4g6Yb7UoPlquFXOSVZiys/EsM+jQKfst1qWnvPrt0UV9l2qWse5cAp5m1XVV4Tq2H27Wj0zVfFT+PHtL5JI7FridDEYJWiEIDGJkdbbYHYCMndcMPmcbQxCoeIdZ1+w14LFLdwwHU7G3SNbNXt3tJJIVeRpSPllMjvHtDE7cHyxnzRTjvfEt3HaJNHfXMjS2b6lDdaaqx2c4u7fKW5MY8yMKbg78ybREjBwcM0lte+NLfRNLmuZru5uL7TEnviLBVaxYPbq/loFz5vlSTvsbfueL5UAzGdjwlfanq9pr8F7e3YeC9NtazyW0cM6RG3hdHKYIDneX+ZRyeUT/Vry+kXPiPTvB2iWtu2sxTwaPCLOA6bn7ReDerW9zmLMUSlYVDEx5VmO9sbl6DxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdMtNZ8cDxHp0d/bfZVf7KJYYoZZYCHRPOLGO3kwwcyqpM6AbELArln2PFN/rttqMi6e98m20V9PjtbMTR3l1l8xXDbG8qPiH5t0XEjndxlSK/127vINJL31vdLqF2bu4WzARLMifyCkrIYmYbrY4BZuDuHD1X0A6pp/wj0ZLZb4X0NpbxTCa12zwIGVZtsZQbmjTfsG1t2xeJM/NlxNr9/qej3WoT6yLez8R7IG/s1UZ7ZrLCSSKEYqC7MrN8oHmvlYyAse54in8Sx6jrM2mTzpa2GlR3drbx2iyfa7oGcmLcQSVISMMq/Nym1k53c3rWseKNH0LWbi3F3Ym3S/nsxBpqNHMwubks02VAQLEIZFYld5cn98fkOxo9pr9mY0F1dxW99rGpQNClqo+ywtJdSpcZZSS5dUwzfuysijYT8xk0A6pp/wj0ZLZb4X0NpbxTCa12zwIGVZtsZQbmjTfsG1t2xeJM/Nz8viHxjIl2kEt89rBqDw/aBZ+Xcqn2e3eENGlvORvEkrtmJecA+VxFUmsWGtalb+J5ZptS+16j4SgKxWtiRA8yibfEm+LeDlgAjHzMTNxkLs1LBPEV54lmtbe9vrLTJvMle+bTY45pttvY+WxLRhRIWaYHch4V12jYuynJ4j8V7NLurtbuxnuLjTEltVsB9nEEzQCVmd+Ul815Itu4sFAPlj/WjU1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+KvpdpqK+NdIu7kX0EP/E5h8qKyVYebwNGXIj+Xeg37iw3GNSCSzbxZNds/G2sC1jvlhuNVtnhj+xA29zGYLdLhnl527EVivMYLggGU5Rdjw3faze6jJa38ny6TEbS7fan+l3JIKyYH+r/AHQjk2jI/wBK25zGaz31HXReajsuNVOrx/avsemHTx/Z8m0P9n3XHlfxKI2P74fMxX5fujPt9T8Yy69FbafLfXGmtLAkV1qumeRlGiuzK8gVFJ2ukOBtizhF4D+Y+hc3euxeNLPT473VZ7KPykkMVoIw+ACxkdrYxPnqxSaPglVTeo37Gq63t8PWXiTTrjfpkfl3tx8mPNs2Q7m5G5disJcAbj5e3HzVz99f+LreBDM99FfPafaLO2tbNJ45bp3kb7LcSKjBY4x5EfmbotwLtu/ualnqetS6pZ6dLHdieLU7p712tCsRsv3/AJG2XbsY/NbcIxfru6Pivc/8JJP4kuYo9Q1K2sZtTFmnlWsJWCAWazGZGaI5JmUxlmLL8xUANgjHsNU8Vyxm7nsrvT3unSdhaaYGZ7k2doUidWGfKLm4VnZlK+WqmWPFR+E5/EsulnxAZ55GvbvTJHt1tF23QmtrOOeZjjO1QXI2bQrI27cPlG54ai1fTPBmuR2v2u81aG91FraPUIBEJJDLI0eCFjDI+VbcDjLsAQBhcebWfE0aamsF3rMlmqWosbm40do55SzT+b9yBypBVBuaADaFG0F1lbpFutZj8E2d2v2575ZYJbjz4kaZofPUzYRY1z+637R5ayYwNofisvVdW8aGKRrTTJIJIreV5U2KQFeeHymVgJN0qQfaNyqH/eJ9x1aPfz5HiHWbnR7i+tru4NvcBBKbWZXCfbtNk/eFoIQThZTlYwNqdyrGuk1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+KnqF740n1Y2enTXcE81xeRP5lgv2W1UQz/ZHWXaSQxETv8A6zDfKTH/AKt7Gv6t4re3mvtLtLu1smeCBopIh9ohKiYyyKEimJBcwR5CSAhXZQEIlOhDqWst8PDezySJqmxlDpZTs/8ArCqnyzCH3lcZbydoJL+WUG049tqOvnS7J7+88QW1tI85mu4LBbi4DjyxEixi1VhEQZiWeBTuQfNtKl5LKTxle3cX9oXV9p7XN3FazxW1tC0dqv2FJnkjZkfP+kBotzM64YgfNginaa/42uNb0l7y0ks4rhLNntxazGIrIkZm3hbdyjq7SqN08e3YhZcZL3PEkmu2XjLULrSo74eZpVstsYLISx3F0klyUgkc52Rnf8xAXAKkyR8b49Q1XxpHqxTToLuef7ReK9pJZKLVFWGc2m2b5SRIViL/ADNhjtLRfcaO8l1S4k0ufSLnXNYurbUDJt1bTPsqxA2l0u5f3MIblgMFgC2xS0e7dWouq60rvYRHUpp572yksp5rAqGstsBnMj+WEjc4ucq+1wSAqjKCqfgbX9d1Cz0y71WS+uY7nT45LsyaeI0imcxCHySgO9XV3d23MF2gkQjKVsarodvofgPXrXQbD94+nyCKDYbjzHS2EUa7H3bvljjXaQc45BJOa/iwapDr+n3umQTny9PuY5ZoofMMKPcWe9lGCDII1lZVw24p91vunP8A7T10xRie91yHSPNlEWpQ6UHvpcLEUEkHkHapZrkZ8leIo+fmy8nhV/Ea6/dNrFlJZJd3Ec03kRb0kuPsNuGQtzsiDLJhgSGZNpdcBZZNb1HxLD4h8qz+3CP+0LSNIYbFZYGsmeLzZmkI+WTcZUIDEhBu8sD96Kdvp18nwr8X6cx1K7vGfVkRLi22PIXeUpsCoocOGDZUEEuQMABRH4mm8XWEksNqZ9Rmgia7gu105G8udrS+BSIBThVeOAKCWf8Ae7Wdg4B0DqWs6DqE1xqV9fXGkWd28E809qmPsq2ZuDckxxqdwk/dZX5cDbtL/NWh4r0iXVda8JuGnMFrqpnlRIUdBtglZXbcjbcMFUEEY8zj5tpXk7LXfHzWuo+bbyfbBbh5IXglxA/mxiTyH+yqhCo0pA3XJYqm3zADvjhl8Saje6RdXdrPctBKYfMksZCwjN5pzEyeZbxDcAJmDLGoCpkHcjNWpbXvjS30TS5rma7ubi+0xJ74iwVWsWD26v5aBc+b5Uk77G37ni+VAMxnU8M6vql1pfiN7y5neSxu2jtJLiwxIIvs0MiM8MZDMx3lio2sS2NsZwi83Mdf1XwjcS3l14gDwanYSgraK7bFuELlIzZxSMVHzn92R8i4Zv3i1qaxrPimPSbu1sIr46vay6hM5WwJRrcR3BtQjlPLkbJtflUl+oYcPVi7Pi6LXrXSYtRne3mlhkn1AWCFY1eK78xE42qqtFAU3F2VpF3mQEKcfVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNly6utafxdpc17DqUlvZandTXEUenlobeJbe5WBo3UbnLoQWA8z52C5jOEbQ8YW18/ibQrizs5J0jTZIwt/NVAb6wJzkEA7FkYHqNhYY25HP6fdeL9N8AaXBFa3dtcQW9lDLElu2YLcQH50Pkys0pkCq6mNig42Afvm3NRXWNb+D2qpcGf+05tPuUAgt3WSXG8KpSSJGLOoCtiNcliUC5XFewi120vtT1O1uL6W3l1WxCQTWIR7qOSG0ilmkyoYbV3HCCMKyPuyPlWnod94jF14bspP7SghWytBc21tp/2dI28pS6tvtjHgHOds0eB8ipvX59TxRd6xZatqcunieFXtLJI5oLB5XdvMuS6iRIZSMDYfmR1G7ACmQMND7ZqkPw/+1NLfLqRizvntvNmyW7JFFkZB+UmHKjDPGSGSsO21HXzpdk9/eeILa2keczXcFgtxcBx5YiRYxaqwiIMxLPAp3IPm2lS8llJ4yvbuL+0Lq+09rm7itZ4ra2haO1X7CkzyRsyPn/SA0W5mdcMQPmwRTtNf8bXGt6S95aSWcVwlmz24tZjEVkSMzbwtu5R1dpVG6ePbsQsuMl9jxff61BrNjbaZPqSxOgMiWFoSxJbH+te3liJ44VmhC9WYhgUz9W1XxXENVNidSfUES9xaCwBt4YljlNvJDJ5f7yVmW3ym9+ZHGwYwmhrNjq7aj4VvLy8uy6awZJ4bC3EsUCtbyoqk+WX2A4QyHH+tdvk+QRyeKb/XbbUZF0975Ntor6fHa2YmjvLrL5iuG2N5UfEPzbouJHO7jK2PDOkS2PiXxdfTtO7XmoReW80KLujW3ixtZUBZQWZBkkDZ/eLlubtb7xHqEk5n/tK7jW9syqy6f5cAU3kW5hHLbRyKQgZhh5tgBLOGCM1dtd8aNqFjZxT3aXdxbpJcR3GmKUguPtdokyJja0kEaSv8w4wxPmuRmPuNCuL6O6v9N1CS7uDb3BW2u57faZ4vKhcszIqx5DysowBkIeCVY1xdrJ4utYNVtNKjvku2u9Va3gmskSBY2e4eGdZWyGkMxjUBmA2Nny9o803PE9zqWvWd3cWTazb6Xb3GlvGI9NZZzIl0WnIikiMjBUML5CkZTAzh1PQeKLvV7a4sl083aQMkpL2lqLhnuAU8mJ1PAiYGTcxKAbV/eR555vS9W8U3Wp6BFeTaqC9patexRWJhAkMatIJDJbGM8k7is0ZAyqpvX5+wthcf8JlqZaDFqdPtBHN5IG5/Mudy78ZbA2Hbk7d2cDcc8HdSeI9e8NeLbaa41mXfpk4t1Sx8lHYg4RY5bVJAWAKhVebALAuG2M2pean4lTUbZLOXVZIzd2KIraYuyWyYw+bPKxRfLm3NKrKOVUZ8pR+8WOW98bx+BdM1GZpP7QuUtxdxQwkSWqeUxZ8CCRjK0mwMoiYKDtCja0p2INQ16TwDFdsJDqRcLK8ds4lSHztryCOSNGaVYctjywGcfKhBCnn4m1+/1PR7rUJ9ZFvZ+I9kDf2aqM9s1lhJJFCMVBdmVm+UDzXysZAWPU1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+LL1/WPGkEt1/Zw1Izh74PDHpqvDEqQTm0Mcm3LlysTMP3nz4UmPPlvc16y13+3dEu5oftUx/wBasNsJYbZWvtPOxSV5wiSNvbDZV3GwAKljQdU13T4NDuPEl3PKuo6ejXXmWQiW1ume3jijAVcqzmZtwcn5kJUIoKg1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+LLur7xHbyQWtt/aVuHvbwtFYaf5bMpvJQjb2tpISWTaRuaLrvd2D7l6Dxfca/b3EcujSXYS20y9vDFDbrItzPGYfJhfKk4bdINqFWPODxWfdXmv2l1LY3l9rMem29xJGup2mnLPdTHyoHjBRYWUoTJcKWWMAGJAWBzvktdX8UmJbO/tZ4dXuLuxlRIbUvBHb7bc3S+aA0a4IugAz7+m3OUzj6BomsnWLWBdR1W28mXWlnvJrFA4D3cLoIy0Yj/AHgG/dtcHMgXGBs6S61PWpvCGh3ssd3Yz3SQvqjWVoZJ7UNCWPlxMrknzdiEFXIVmPGNwz7jV9atrC+jsptZvHutM2aRPcaWVkN7vnDeaBCoiHNuAZFVCBnn5jVy2bxdcafeXbS7JINQlW3tBGivcW6XhJJLqAGaBfLjGQPm3s5LAx5fiDVfF6BbqyS7h026uJHiMcDfaIYxHEsasi207AOwuJPmjyAUDMh/dmvqN14m1I6fHdw3Zdr3S5Xt7XT2+zGASW8kkxdwHRxKJF2MSwRQTEOZBsa0NfbxvFBpjSWlpcJaxT3iWSyYTZfM43kdQwhxkkKzr8pDlWuG910eDppCk/2+K7e3aUQAzG3S5MZnVMYaTyAZAApDNjCEEKa+i3/iWXQPEcpinub6CVxpC31qtsZ0FvG0RZQR95y24kryWBEeNi0zqN8LIm01jxPcQG4Rby4uNF8ue2i2SENBH9mXzCZBGrfJJtUk4X7wrm98aSWup3azXaJY6Z9qs4TYLvvpVlufLV8rkF4kgMkahXBcbfK5U17nS9c0zxDreqaXaSX1/cawzQfarSLaVTS3MWJAqkJ5rCLduHQrkFnLU7i78Uwajql9po1W4t5IrGAX93YGC4MKm7ZiEWFyWEjIv/HvnYwO3GJjoQaj41uNMvr37ROrWOlC6ggTT2BvLhZLkCNjJEjHciRB1VEJLKU8sHDdB4Ztri18S+LhcTX0izahFNF58AWMobeIZjcIA2CpQ8nAjXPJJbk28QeNE1Cxtovtcl3NbpLcWtxZLEiXAu7RJokkCFmgRJXHmBWADFvMkI+TYvIfFEXiy2tLe8kCXCWyXepR6chG0rfu4U4O0KwhCBi2wsm7fvbf0HhzVJrqKawv3kOpWjyeYJIijPD580UMp4C5dYS3y4HcAAgVjxz6va+APDPyXdk7W9uuovZWIae1XyCT5cARgD5ojQqIztVm4GNy8/p114rsdHZHh1K0vC93cWdvDp4ZLu7e7uGMc5xIIoseRhvMQYkYiRgNy7kcGu6jtg1h57rTtS1C/wBOuLI2ojSKz/0jy5CygPuISNQ+4KVdflLHedDwzbXFr4l8XC4mvpFm1CKaLz4AsZQ28QzG4QBsFSh5OBGueSS2PbanrraZLI97rh1fyozeWb6UFgtMyRi4Nu/kDzWjUybBvl3YHEnePSL3xpqGtvZtNdxae73KreXdgsLrCEtDFIF2kGVt8+A20ZLlkHl+SLGmar4qfx49pfJJHYtcToYjBK0QhAYxMjrbbA7ARk7rhh8zjaGIVJNb1HxLD4h8qz+3CP8AtC0jSGGxWWBrJni82ZpCPlk3GVCAxIQbvLA/ejD8NXXiux8J6daJDqSmDTII7yObTxG1iFaFAYBg+a/kGdyMyHfGoKIT5R3NP1vWLO8iuL/+1brQh9oiinfTHNzM2LcxtJDHEHTDfalB8tVwq5ySrMeFrbWYfFrXGtzXzXdx4fsBOpgTyPPRpRKokRNu5WbIXdz5rHBULt+OKK+/6KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK//2Q==",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAAK60lEQVR4Ae1ai3IcNwzzZfz/v+wSAF9a79nX1EzUGV3cFR8gSJFrKdf28fHWP49V7a5FXnCL8hy2eEx5HnaLXOHSms3ET7YrD/Rfq/HF/f5A0ErxgraWtmoWnoYU7jkvG3682Z8HoQ9bIUKjxT0ww+RueU0jBhEhyCIHzeZoPgLpTWJ5SW7INJsBtjIoSed3t3CoLRJZEI2w8J936v2R7wbfkfbS4JV5fOBBeHcnKr1hwVpBsupJJnqTGAI+tiAHlsrXDMA4CBEeBWh4UvD9ygO+zxv2KIXY03iC0W0fYkkEMPVOXbyBhTl4GpokShlImRomGYlrucjZynCxIVjZUt/nDX8mCEt2MAy1WilPvNE/bCfkhi6K9NLUd99lsXiuztmIwHSppiwvbDgGjG598HGpwOhVB1uJ/qqnSEztA5EVFOgH7LlPIVloh3eZm4poORRFR+4yEFVrWhLjEfdL9NPqH//EZMYS1SmNU9A+fKzpfKO534KUpAjoV9vKdad5ZroyyR0wyRmhs/oeBysLsYef2A6sDbvhm4yO+uHl95La79e3dQCxoH5xKK3HEtkYPPBDCKl5wVV7IbmTN590xFRXcw4UEq1IQo1ZK8OBgMW5GMAsjdO2gIvLzVpdi9xRgKOYwiBvj3fdOp5GyZTR+oJfqP5LJRmHGH5wIkXzIOAMwqFGb8RBC75MEkZBaXaTLMwKM1W57BmcqlLJ5ZTF8ViaWThHPD78lY7CIzJX5LGTEAuEaByUxEDonouWwBQ8WqRJBHeDdBXymsEM8akYEgauzATGJvxayuab01ripUAJWkWhe27pITe4iktgCuaD7ExhjvWGv6GLN6SKI+F9fQF+w4Z1o6UJO6YxKpLB/JGXFyhvwLhKqVgUImXzoOSWANxyl4c/Vq+iq5LtyRZlw4Usp29UOJmdLJDc1SV+gZTyGqrwT6Tfp/ky8ktnlSJYu5bq1SiQS2zbau3oLq8ovRVhW2giyNdQA2srTPYPFh+fnIK6x/yLU5C7p2Btw3egsL3IGfBX1pcpnwCfmL9L/QvnLlqGvydLjGNOd6FOR15nhPHyAxzgPLQppEpBpA4XV8VGJrDAJz7J0MKECP5xHBah3NFUkMjJaAXCZmwR9/b2rluTSHvY64E3hG8JZImp4jhTLKVAKgyoQEowQj/wFeeK4EoWORecmGBqZohRt6p0Z9D5HhCc3BefOdZXmuDsx/qrQx6LiBW08ZGtBYajrS2uREilEQ31yhQQX0N1dqp2PmirbhQvTO1vDwZavx4qNEOio2Uwybu82q6Bi1fKXdwdGXCke5XjutGISyJjq168+3fcysGjFBlN0E/5wgDOuIIhA4eHSoUgFTTM5UQtsUchRp+GCyaYqtTgiABb5YQDNSAba7FHA6usgBYf0dfHnfvOdo17Wb8huzEl3e/6ojdvj/V3OIkhxMB8CDkLNXKBuoKD0QPv3He2TuYZmilzRijOrcrxyR0wrHLm01njy0NHhtwSh2mf9d8V19C/0AkeiugF+wFN9yAWmoGR002GEZYtgFu6d4QKgxne3AokGg8Hkgv5CWcqASMvTHTicECSilCIdH8SHRCEIdgJcErjF6P/cvCWcyPpYGg34opm5Eph2vKVtDIQHKrnqdgWJ2DPRCf22j+K1TPAANYZFmGe7O0d4C8+ODAvCD8AVysak59wIVifEG7orM5A3cSF6VMVEdPW2kolvhZ/uYdbtItZTLlgsjqc3r88V2EF9EYDmO4KLFw6o+JKGneNgcNZcVepwEVwxeCvlrxA695sEJ+lW5hRJt8zQhGo4NoVAzyYVbDj2JcCoeYumV4qQqDqhxgLdzSd113L5whWwu3AcN+hx5XBN1fLt4CCjko/VIffwzoAs2CccvGpScBS9vAvq7t1Sl49cjaGJi7YW2WpY4nsxS8O8Symp/8Sb6G/LeBr4/P45mni12zfeBtPE58E8W9anCjvRt5XBuWM0BtcXwzlvRijC7etAjk0kzDGHiIVH8nARiomSdEFwvFotCwCupziIn9YUkFZCOUfWVERg5nPHPy2hIMFvyM6cf23hTY8EOMnAE8g6jDEj60KrRweIo8pDBTeqcgCfxMc7swZLFSotnoZsjQFVGZEGVicvImGra+HvjHmxyNq4YmHgzMthMitZ2xUUGdgvvC0kE6VrCl4MJcIXpyxjQ6E3EBNpGeB1ob7DjrE05r7Wa7sRFTIcHZepJ3O5EaVYgoNm8HdmcYGpFigpYxr2blhtIVfI5dJmpUOvibIlddiy5ehJrSm2I7lIbQcTKTwFCUAXhvK4Bsn2RZ4UJmRgUignLDkC/D9PRxhqvD/8azmfq737vswmhWfpedhzFXIjqcLp2T7PEE1BERDLVEX96oS+RRebwmJoxzi7za84tdML2o3FDemF8mewV5ltBe6KLRh3l241lrnqLE5frNxBkLYMOCJ1vGmCxPnRF9NAFH4iQSIhpc6BeeiVQ6R8xmi8AoFCH8QDVmYyOHE8BJFAGE8tOydh6f1ITQctfLSAghhsvEgBg1AMkVgwlmNLkVGQreOR4joYIKXVlMKCXRqggAEMz6QUXbaqNEmV3yRF8CecUr74VZ7xlGXGhRSZSZUyJR4BC5h3Vj+hRKQT58gWhxMLEukzNwtIG0V3LxmZHW5YVCZv5F3zb21NUNH8uKnaU3SnJ8SrL7nGhht/igtyG9y39WzcmoTnLDt3X5yV8SFxss0usdVBWjCvUHtAsxMza8WRclEgK5dkeVTHrnFpf2yKDPTn0lMSBs98RW6IQSw51JRA/yQOEz/G1W2ipooosWQSgoEmbYantVwg6IJQ+ufGnO3/px8dw8H+5J7UQLxP1zx/2nZi60n6+eN5ro8MofT1nbL4YrQkMyIKKiMFgpRtHHJUBOEI+uffOD/08JLGfcVZVbJkcqZBUG1Ws0lByQeJzLK7NGOShsj0mggD0zyPyPw/8S7vK5QUZl9Lp7F8tmXIYzmxGkizRO4AH/uiX9Ne/3gFb0xCwbnl5+MbhSweQu/jP0DTv73Yc/DkvJGq+RVat5mS0MiztrUok1U39yGhRdoPP9OD2oztcHrMG4xDf6fxOsb096L/8T7LHh0M8+S/k37V/fw36xrLPfZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4jPhTQYxVsaZ8FhrNyE+E95kEGNlnAmPtXYT4n8AEZjD5aBTZrMAAAAASUVORK5CYII=",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated[0][0].to_PIL()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCADAAPABAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APf6x/EouDpUAtYPOk/tCyJXyRLhPtUW9sEHGF3Nu/hxuBBGRj+FJ/Ev/ElOtTz3LX+lG4uxJaLCtrOvkhUAABVmEkhYMTlkJUIAVrLubrxhDFfXUVzqUjqmp3cNubKMrut59ttAMR7ikqNk8722Aoy853LO+1mTxQ2jyyfu7OWS7muNqfvraQMLePb/AA/MZF3A5/0TJGJRWW2s6/J4z1O0t5buSC21O3gt4Es1MDxNFA1x5soBKlFdnXJTLHAMv3Fx5brxXqVzpttdw6lLAXtbi+jfTwiW92l9Zl443ABaJF8wq2CCNx8yTB2aGkXXjCO106e7udSuJPs+mTzxT2Uah3uZfLuY2CxqQIVXeACCpYlyy4AueKLvWLLVtTl08Twq9pZJHNBYPK7t5lyXUSJDKRgbD8yOo3YAUyBhoC61lPAkNwn25tQbYZnmiRp0iMo8x1RY1BZYyzIpjDHCho92Uqvot/4ll0DxHKYp7m+glcaQt9arbGdBbxtEWUEfectuJK8lgRHjYuPey6pcCxuNGudc1i+tLuSWJNX0z7KiP9iuwnPkw5VnKKSTxleVzzHpus+KpbKZbufUktFuIhNdR6fLJdwxlJclFeziVxvWBcLFIVDuSQNpXQ8LXfjTUtQaPVzJZ26IJZWktVjcsLu6HloPmXDRpDuO98KBtJMnmCx4vv8AWoNZsbbTJ9SWJ0BkSwtCWJLY/wBa9vLETxwrNCF6sxDApnkavpmseJTHdaylvcaxFJNcRWAleK3NouHgAiIkPmokTACQqq5IBy5De+NJLXU7tZrtEsdM+1WcJsF330qy3Plq+VyC8SQGSNQrguNvlcqSPxH4h0/xZey60ZLbR7d7l5gbeZ4kto1dkkRkttocqqMczsPmcbQxCr0Hii71e2uLJdPN2kDJKS9pai4Z7gFPJidTwImBk3MSgG1f3keeef0K98aS6bDqOoTXfmR3GnRGxawVPMWWK2Fw7/LuwjSTMNu3ayNuLKNqyeG5NdvfGWn3Wqx3x8vSrlbkz2Qijt7p5LYvBG4xvjGz5SQ2QGIkk52V0m8XRR6/NbGe2hsIrm7tLSPTkH2yf7XeFUzt+ZXRYSwUb23owdSxLx65feI7q68SW0H9pXUCWV2ILddPxAzCJgisk1t+8G7A+SaTe2CECFghrFx4rm121ktbS7ubu0vb2WO2kswtrERbXKWpWbK5Eishf5nAZsFojhG3PAuo63ffb01d55I4/LMLXNvLHJk7twy1rbqV4XAVSQS2TgqBn+B7TUYNcgnvxfQtP4a04GA2SxQRyJ5geMFYwEZSwITIP71uCAoSndX3iO3kgtbb+0rcPe3haKw0/wAtmU3koRt7W0kJLJtI3NF13u7B9y9R4ou9XtriyXTzdpAySkvaWouGe4BTyYnU8CJgZNzEoBtX95HnnPvNS16P4c6PdNJdw6xNb25utlk7S+YYwXG1IZfKO7OSYmHBXClgy58dx4tlEUi3msxoLexXZLawMXae5ljmkJ8lTmOLY6jam0BTInLLVfWNY8aQ6Te/ZxqS3lvb3CWHkaas32ySOadC0wK4U+UkMi4KBmkO1ZBiOtTwTYXlp4i16W5tJ4Y5d3lvJGVD/wDEw1B+CevyujfR1PQio/DsviiPQdLsJreSG4mSwmjcWyRRW1uscH2mF1CjY+5ZQF25/fJtwEfy6aat4x+zRNbLfXF832c39vdWfkw2s5uYFaKFxF88JRrgNIPO2rGrbu7bGrXHiLS5YLGO++1TarF9lt7kRRxi1utzMzqhzuURM7hWPS0ClmaTJk8cX+q2VvZjSp7uN3di4tbSSRiABjLrb3AUc/dKAtkEMNpDY86a5Y+I9SvIZdZaSfR9PB820ik8tVnkFwQ0cZRp40ZnCAtuaQ4V1CqK8Ta/f6no91qE+si3s/EeyBv7NVGe2aywkkihGKguzKzfKB5r5WMgLHYl1rxfJoloLO2u21C00zZqvnWbRD7UXgDNGxjIkKoLph5SyLkL8r5RWr2smvakfDN5qNxqQ+ya6ykxWLlzE1s4DTb7WM43sU3qioFk5O4bxTWXxTouii1spNVeP+1dRNzcPalZgxnLRbQlpLujdWeQsI9u7gOowh7yPUdVuPCE1zbW0cuuR2W4QtBJDFJcmEOFUS7W2FmA5II5BIYEDP8ACl7q9zrOqRXc2pXGlx29u1lcahYC2kkYtN5uQFUkghByiYAX5Tne/Nxa/wCLl1SygkkvizWn2q+gfT0Xy5EubUXMcJxuljSJ5ApUMW3fLJK2Ampdax4jnupbi2GpQ6ZNcSGxZNN3SuyxQCOKSJ1DJE8n2osz+X91f3iKVJj0C98c3mhXt3cpi7trSOS1t7iARvdXDWMTFHBChYxMz5wcljtyioQ2OX8SnxLPf6baX2pMsUK2VxqmmrA0pS31EgNwmzErom7bEdrqOQ299C3vvF15r0VlaX+qjSZJYES/vdJSOQxmK7MpIKrhg6QqCyIAfLyjhsy9Ab3XR4OmkKT/AG+K7e3aUQAzG3S5MZnVMYaTyAZAApDNjCEEKcPSJ/GGq629nNqGpWem77nZdyaZHHK8SpaGEjcpVXYvMTuXn94NiEAR15PEfivZpd1drd2M9xcaYktqtgPs4gmaASszvykvmvJFt3FgoB8sf60U7W+8YaqVtdYt5ywu7GeeD7NMRDIl7blhE/2aNPLVfMJ/ezHCqQxAZm9E1DStM8TabCt/bSTW7ozoknmQsBJE8bZX5WBKSuMHBGexHHD3Ol65pniHW9U0u0kvr+41hmg+1WkW0qmluYsSBVITzWEW7cOhXILOWksfEOuxeI5Ea51W80W0+zSyNPpYS5eGRLxSxiRBIf3yQrxGhwmdpXdI+podx4r1DXJku5JLbS9i3KSSW4WRkN1clI1BXgtCLffu+ZAMbQ7l1w9L1vxxIt4t9HPHN5SNcILSVzbHzYxKbcm2RG2RtMQu+4LFE27xu33LOPV28HePJntbue7uHmez+1WAV7pRZQpGxhIIYttwV2jJyCiHKLueGdIlsfEvi6+nad2vNQi8t5oUXdGtvFjayoCygsyDJIGz+8XLZa6p4oltbqFXk83SUSzvpYokke4kaVC86IAdrrbATKgDgm4CYYxlTJp+t6xZ3kVxf/2rdaEPtEUU76Y5uZmxbmNpIY4g6Yb7UoPlquFXOSVZiys/EsM+jQKfst1qWnvPrt0UV9l2qWse5cAp5m1XVV4Tq2H27Wj0zVfFT+PHtL5JI7FridDEYJWiEIDGJkdbbYHYCMndcMPmcbQxCoeIdZ1+w14LFLdwwHU7G3SNbNXt3tJJIVeRpSPllMjvHtDE7cHyxnzRTjvfEt3HaJNHfXMjS2b6lDdaaqx2c4u7fKW5MY8yMKbg78ybREjBwcM0lte+NLfRNLmuZru5uL7TEnviLBVaxYPbq/loFz5vlSTvsbfueL5UAzGdjwlfanq9pr8F7e3YeC9NtazyW0cM6RG3hdHKYIDneX+ZRyeUT/Vry+kXPiPTvB2iWtu2sxTwaPCLOA6bn7ReDerW9zmLMUSlYVDEx5VmO9sbl6DxcL6LxVot1btqUFotleRXV3p9l9pkhDPbFQFw3LMmPuOcbvlABdMtNZ8cDxHp0d/bfZVf7KJYYoZZYCHRPOLGO3kwwcyqpM6AbELArln2PFN/rttqMi6e98m20V9PjtbMTR3l1l8xXDbG8qPiH5t0XEjndxlSK/127vINJL31vdLqF2bu4WzARLMifyCkrIYmYbrY4BZuDuHD1X0A6pp/wj0ZLZb4X0NpbxTCa12zwIGVZtsZQbmjTfsG1t2xeJM/NlxNr9/qej3WoT6yLez8R7IG/s1UZ7ZrLCSSKEYqC7MrN8oHmvlYyAse54in8Sx6jrM2mTzpa2GlR3drbx2iyfa7oGcmLcQSVISMMq/Nym1k53c3rWseKNH0LWbi3F3Ym3S/nsxBpqNHMwubks02VAQLEIZFYld5cn98fkOxo9pr9mY0F1dxW99rGpQNClqo+ywtJdSpcZZSS5dUwzfuysijYT8xk0A6pp/wj0ZLZb4X0NpbxTCa12zwIGVZtsZQbmjTfsG1t2xeJM/Nz8viHxjIl2kEt89rBqDw/aBZ+Xcqn2e3eENGlvORvEkrtmJecA+VxFUmsWGtalb+J5ZptS+16j4SgKxWtiRA8yibfEm+LeDlgAjHzMTNxkLs1LBPEV54lmtbe9vrLTJvMle+bTY45pttvY+WxLRhRIWaYHch4V12jYuynJ4j8V7NLurtbuxnuLjTEltVsB9nEEzQCVmd+Ul815Itu4sFAPlj/WjU1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+KvpdpqK+NdIu7kX0EP/E5h8qKyVYebwNGXIj+Xeg37iw3GNSCSzbxZNds/G2sC1jvlhuNVtnhj+xA29zGYLdLhnl527EVivMYLggGU5Rdjw3faze6jJa38ny6TEbS7fan+l3JIKyYH+r/AHQjk2jI/wBK25zGaz31HXReajsuNVOrx/avsemHTx/Z8m0P9n3XHlfxKI2P74fMxX5fujPt9T8Yy69FbafLfXGmtLAkV1qumeRlGiuzK8gVFJ2ukOBtizhF4D+Y+hc3euxeNLPT473VZ7KPykkMVoIw+ACxkdrYxPnqxSaPglVTeo37Gq63t8PWXiTTrjfpkfl3tx8mPNs2Q7m5G5disJcAbj5e3HzVz99f+LreBDM99FfPafaLO2tbNJ45bp3kb7LcSKjBY4x5EfmbotwLtu/ualnqetS6pZ6dLHdieLU7p712tCsRsv3/AJG2XbsY/NbcIxfru6Pivc/8JJP4kuYo9Q1K2sZtTFmnlWsJWCAWazGZGaI5JmUxlmLL8xUANgjHsNU8Vyxm7nsrvT3unSdhaaYGZ7k2doUidWGfKLm4VnZlK+WqmWPFR+E5/EsulnxAZ55GvbvTJHt1tF23QmtrOOeZjjO1QXI2bQrI27cPlG54ai1fTPBmuR2v2u81aG91FraPUIBEJJDLI0eCFjDI+VbcDjLsAQBhcebWfE0aamsF3rMlmqWosbm40do55SzT+b9yBypBVBuaADaFG0F1lbpFutZj8E2d2v2575ZYJbjz4kaZofPUzYRY1z+637R5ayYwNofisvVdW8aGKRrTTJIJIreV5U2KQFeeHymVgJN0qQfaNyqH/eJ9x1aPfz5HiHWbnR7i+tru4NvcBBKbWZXCfbtNk/eFoIQThZTlYwNqdyrGuk1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+KnqF740n1Y2enTXcE81xeRP5lgv2W1UQz/ZHWXaSQxETv8A6zDfKTH/AKt7Gv6t4re3mvtLtLu1smeCBopIh9ohKiYyyKEimJBcwR5CSAhXZQEIlOhDqWst8PDezySJqmxlDpZTs/8ArCqnyzCH3lcZbydoJL+WUG049tqOvnS7J7+88QW1tI85mu4LBbi4DjyxEixi1VhEQZiWeBTuQfNtKl5LKTxle3cX9oXV9p7XN3FazxW1tC0dqv2FJnkjZkfP+kBotzM64YgfNginaa/42uNb0l7y0ks4rhLNntxazGIrIkZm3hbdyjq7SqN08e3YhZcZL3PEkmu2XjLULrSo74eZpVstsYLISx3F0klyUgkc52Rnf8xAXAKkyR8b49Q1XxpHqxTToLuef7ReK9pJZKLVFWGc2m2b5SRIViL/ADNhjtLRfcaO8l1S4k0ufSLnXNYurbUDJt1bTPsqxA2l0u5f3MIblgMFgC2xS0e7dWouq60rvYRHUpp572yksp5rAqGstsBnMj+WEjc4ucq+1wSAqjKCqfgbX9d1Cz0y71WS+uY7nT45LsyaeI0imcxCHySgO9XV3d23MF2gkQjKVsarodvofgPXrXQbD94+nyCKDYbjzHS2EUa7H3bvljjXaQc45BJOa/iwapDr+n3umQTny9PuY5ZoofMMKPcWe9lGCDII1lZVw24p91vunP8A7T10xRie91yHSPNlEWpQ6UHvpcLEUEkHkHapZrkZ8leIo+fmy8nhV/Ea6/dNrFlJZJd3Ec03kRb0kuPsNuGQtzsiDLJhgSGZNpdcBZZNb1HxLD4h8qz+3CP+0LSNIYbFZYGsmeLzZmkI+WTcZUIDEhBu8sD96Kdvp18nwr8X6cx1K7vGfVkRLi22PIXeUpsCoocOGDZUEEuQMABRH4mm8XWEksNqZ9Rmgia7gu105G8udrS+BSIBThVeOAKCWf8Ae7Wdg4B0DqWs6DqE1xqV9fXGkWd28E809qmPsq2ZuDckxxqdwk/dZX5cDbtL/NWh4r0iXVda8JuGnMFrqpnlRIUdBtglZXbcjbcMFUEEY8zj5tpXk7LXfHzWuo+bbyfbBbh5IXglxA/mxiTyH+yqhCo0pA3XJYqm3zADvjhl8Saje6RdXdrPctBKYfMksZCwjN5pzEyeZbxDcAJmDLGoCpkHcjNWpbXvjS30TS5rma7ubi+0xJ74iwVWsWD26v5aBc+b5Uk77G37ni+VAMxnU8M6vql1pfiN7y5neSxu2jtJLiwxIIvs0MiM8MZDMx3lio2sS2NsZwi83Mdf1XwjcS3l14gDwanYSgraK7bFuELlIzZxSMVHzn92R8i4Zv3i1qaxrPimPSbu1sIr46vay6hM5WwJRrcR3BtQjlPLkbJtflUl+oYcPVi7Pi6LXrXSYtRne3mlhkn1AWCFY1eK78xE42qqtFAU3F2VpF3mQEKcfVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8AL5O7zG2nGNly6utafxdpc17DqUlvZandTXEUenlobeJbe5WBo3UbnLoQWA8z52C5jOEbQ8YW18/ibQrizs5J0jTZIwt/NVAb6wJzkEA7FkYHqNhYY25HP6fdeL9N8AaXBFa3dtcQW9lDLElu2YLcQH50Pkys0pkCq6mNig42Afvm3NRXWNb+D2qpcGf+05tPuUAgt3WSXG8KpSSJGLOoCtiNcliUC5XFewi120vtT1O1uL6W3l1WxCQTWIR7qOSG0ilmkyoYbV3HCCMKyPuyPlWnod94jF14bspP7SghWytBc21tp/2dI28pS6tvtjHgHOds0eB8ipvX59TxRd6xZatqcunieFXtLJI5oLB5XdvMuS6iRIZSMDYfmR1G7ACmQMND7ZqkPw/+1NLfLqRizvntvNmyW7JFFkZB+UmHKjDPGSGSsO21HXzpdk9/eeILa2keczXcFgtxcBx5YiRYxaqwiIMxLPAp3IPm2lS8llJ4yvbuL+0Lq+09rm7itZ4ra2haO1X7CkzyRsyPn/SA0W5mdcMQPmwRTtNf8bXGt6S95aSWcVwlmz24tZjEVkSMzbwtu5R1dpVG6ePbsQsuMl9jxff61BrNjbaZPqSxOgMiWFoSxJbH+te3liJ44VmhC9WYhgUz9W1XxXENVNidSfUES9xaCwBt4YljlNvJDJ5f7yVmW3ym9+ZHGwYwmhrNjq7aj4VvLy8uy6awZJ4bC3EsUCtbyoqk+WX2A4QyHH+tdvk+QRyeKb/XbbUZF0975Ntor6fHa2YmjvLrL5iuG2N5UfEPzbouJHO7jK2PDOkS2PiXxdfTtO7XmoReW80KLujW3ixtZUBZQWZBkkDZ/eLlubtb7xHqEk5n/tK7jW9syqy6f5cAU3kW5hHLbRyKQgZhh5tgBLOGCM1dtd8aNqFjZxT3aXdxbpJcR3GmKUguPtdokyJja0kEaSv8w4wxPmuRmPuNCuL6O6v9N1CS7uDb3BW2u57faZ4vKhcszIqx5DysowBkIeCVY1xdrJ4utYNVtNKjvku2u9Va3gmskSBY2e4eGdZWyGkMxjUBmA2Nny9o803PE9zqWvWd3cWTazb6Xb3GlvGI9NZZzIl0WnIikiMjBUML5CkZTAzh1PQeKLvV7a4sl083aQMkpL2lqLhnuAU8mJ1PAiYGTcxKAbV/eR555vS9W8U3Wp6BFeTaqC9patexRWJhAkMatIJDJbGM8k7is0ZAyqpvX5+wthcf8JlqZaDFqdPtBHN5IG5/Mudy78ZbA2Hbk7d2cDcc8HdSeI9e8NeLbaa41mXfpk4t1Sx8lHYg4RY5bVJAWAKhVebALAuG2M2pean4lTUbZLOXVZIzd2KIraYuyWyYw+bPKxRfLm3NKrKOVUZ8pR+8WOW98bx+BdM1GZpP7QuUtxdxQwkSWqeUxZ8CCRjK0mwMoiYKDtCja0p2INQ16TwDFdsJDqRcLK8ds4lSHztryCOSNGaVYctjywGcfKhBCnn4m1+/1PR7rUJ9ZFvZ+I9kDf2aqM9s1lhJJFCMVBdmVm+UDzXysZAWPU1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+LL1/WPGkEt1/Zw1Izh74PDHpqvDEqQTm0Mcm3LlysTMP3nz4UmPPlvc16y13+3dEu5oftUx/wBasNsJYbZWvtPOxSV5wiSNvbDZV3GwAKljQdU13T4NDuPEl3PKuo6ejXXmWQiW1ume3jijAVcqzmZtwcn5kJUIoKg1+DXbfxDr+raU86fZtEt5YYo7USfbJ43umWHJByvIDKoDnemGX+LLur7xHbyQWtt/aVuHvbwtFYaf5bMpvJQjb2tpISWTaRuaLrvd2D7l6Dxfca/b3EcujSXYS20y9vDFDbrItzPGYfJhfKk4bdINqFWPODxWfdXmv2l1LY3l9rMem29xJGup2mnLPdTHyoHjBRYWUoTJcKWWMAGJAWBzvktdX8UmJbO/tZ4dXuLuxlRIbUvBHb7bc3S+aA0a4IugAz7+m3OUzj6BomsnWLWBdR1W28mXWlnvJrFA4D3cLoIy0Yj/AHgG/dtcHMgXGBs6S61PWpvCGh3ssd3Yz3SQvqjWVoZJ7UNCWPlxMrknzdiEFXIVmPGNwz7jV9atrC+jsptZvHutM2aRPcaWVkN7vnDeaBCoiHNuAZFVCBnn5jVy2bxdcafeXbS7JINQlW3tBGivcW6XhJJLqAGaBfLjGQPm3s5LAx5fiDVfF6BbqyS7h026uJHiMcDfaIYxHEsasi207AOwuJPmjyAUDMh/dmvqN14m1I6fHdw3Zdr3S5Xt7XT2+zGASW8kkxdwHRxKJF2MSwRQTEOZBsa0NfbxvFBpjSWlpcJaxT3iWSyYTZfM43kdQwhxkkKzr8pDlWuG910eDppCk/2+K7e3aUQAzG3S5MZnVMYaTyAZAApDNjCEEKa+i3/iWXQPEcpinub6CVxpC31qtsZ0FvG0RZQR95y24kryWBEeNi0zqN8LIm01jxPcQG4Rby4uNF8ue2i2SENBH9mXzCZBGrfJJtUk4X7wrm98aSWup3azXaJY6Z9qs4TYLvvpVlufLV8rkF4kgMkahXBcbfK5U17nS9c0zxDreqaXaSX1/cawzQfarSLaVTS3MWJAqkJ5rCLduHQrkFnLU7i78Uwajql9po1W4t5IrGAX93YGC4MKm7ZiEWFyWEjIv/HvnYwO3GJjoQaj41uNMvr37ROrWOlC6ggTT2BvLhZLkCNjJEjHciRB1VEJLKU8sHDdB4Ztri18S+LhcTX0izahFNF58AWMobeIZjcIA2CpQ8nAjXPJJbk28QeNE1Cxtovtcl3NbpLcWtxZLEiXAu7RJokkCFmgRJXHmBWADFvMkI+TYvIfFEXiy2tLe8kCXCWyXepR6chG0rfu4U4O0KwhCBi2wsm7fvbf0HhzVJrqKawv3kOpWjyeYJIijPD580UMp4C5dYS3y4HcAAgVjxz6va+APDPyXdk7W9uuovZWIae1XyCT5cARgD5ojQqIztVm4GNy8/p114rsdHZHh1K0vC93cWdvDp4ZLu7e7uGMc5xIIoseRhvMQYkYiRgNy7kcGu6jtg1h57rTtS1C/wBOuLI2ojSKz/0jy5CygPuISNQ+4KVdflLHedDwzbXFr4l8XC4mvpFm1CKaLz4AsZQ28QzG4QBsFSh5OBGueSS2PbanrraZLI97rh1fyozeWb6UFgtMyRi4Nu/kDzWjUybBvl3YHEnePSL3xpqGtvZtNdxae73KreXdgsLrCEtDFIF2kGVt8+A20ZLlkHl+SLGmar4qfx49pfJJHYtcToYjBK0QhAYxMjrbbA7ARk7rhh8zjaGIVJNb1HxLD4h8qz+3CP8AtC0jSGGxWWBrJni82ZpCPlk3GVCAxIQbvLA/ejD8NXXiux8J6daJDqSmDTII7yObTxG1iFaFAYBg+a/kGdyMyHfGoKIT5R3NP1vWLO8iuL/+1brQh9oiinfTHNzM2LcxtJDHEHTDfalB8tVwq5ySrMeFrbWYfFrXGtzXzXdx4fsBOpgTyPPRpRKokRNu5WbIXdz5rHBULty5NJ1d/GV2lvd6khbxGtwLiSyDRwwtpjoXjbYFJzmIElgpWPcpJO+PVdb8cWzwiKOdI4fOjimNpK32l47iVAZkitpTtMaQv8vk7vMbacY2dZ4our63uLJY7nUrPT3SUz3OmWX2qcSgp5abPLkwhUykts4KKNwzhsv+0dd/tiBIbjVZP3tqttDLp4WG5tWWPzpp5PKHlTDM52bo+Y0HlnOHw/BVx430y30zSri0kmi+xWhRJrMxR20Ci0RhuyP3u17vcrMWzCpCKDtboNJfxRJ4Y0rVNWvruK/mSySWyt7NH27pIvMaQMqsHIL78ELGrHCsU3N2lFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFf/2Q==",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAADACAAAAAAfRLHjAAALCklEQVR4Ae1ai3IbOQ60Uv7/X/ahu/HiaGSLvqTqcEXFO8Sj0QCBERnv7uPro38eq9pdi7zgFuU1bPGY8jrsFrnCpTWbiU+2Kw/0P6vxzf3+haCV4g1tLW3VLDwNKdxzXjb8+LA/D0IftkKERot7YIbJ3fKaRgwiQpBFDprN0XwE0pvE8pLckGk2A2xlUJLO727hUFsksiAaYeE/n9T7I98NviPtpcEr8/jCg/DuTlR6w4K1gmTVk0z0JjEEfGxBDiyVrxmAcRAiPArQ8KTg+5UHfM8b9iiF2NN4gtFtX2JJBDD1Tl28gYU5eBqaJEoZSJkaJhmJa7nI2cpwsSFY2VLf84afCcKSHQxDrVbKC2/0D9sJuaGLIr009d13WSyeq3M2IjBdqinLGxuOAaNbX3xcKjB61cFWor/qKRJT+0JkBQX6AXvuU0gW2uFd5qYiWg5F0ZG7DETVmpbEeMT9Ev20+v/5JybzzxLVKY1T0D58rOl8o7nfgpSkCOhX28p1p3lmujLJHTDJGaGz+h4HKwuxh5/YDqwNu+GHjI76y8vvktr368c6gFhQfziU1mOJbAwe+CGE1Lzgqr2Q3MmbTzpiqqs5BwqJViShxqyV4UDA4lwMYJbGaVvAxeVmra5F7ijAUUxhkI/Hp24dT6Nkymh9wReqf6kk4xDDD06kaB4EnEE41OiNOGjBl0nCKCjNbpKFWWGmKpc9g1NVKrmcsjgeSzML54jHl7/SUXhE5oo8dhJigRCNg5IYCN1z0RKYgkeLNIngbpCuQl4zmCE+FUPCwJWZwNiEX0vZfHNaS7wUKEGrKHTPLT3kBldxCUzBfJCdKcyx3vA3dPGGVHEkvK8vwB/YsG60NGHHNEZFMpg/8vIC5Q0YVykVi0KkbB6U3BKAW+7y8MfqVXRVsj3Zomy4kOX0jQons5MFkru6xC+QUt5DFf6F9HuabyO/dVYpgrVrqV6NArnEtq3Wju7yitJbEbaFJoJ8DTWwtsJk/2Dx8ckpqHvMvzgFuXsK1jZ8Bwrbm5wBf2d9m/IF8IX5p9R/cO6iZfh7ssQ45nQX6nTkdUYYLz/AAc5Dm0KqFETqcHFVbGQCC3zikwwtTIjgH8dhEcodTQWJnIxWIGzGFnEfH5+6NYm0h70eeEP4lkCWmCqOM8VSCqTCgAqkBCP0A19xrgiuZJFzwYkJpmaGGHWrSncGne8Bwcl98ZljfaUJzn6sXx3yWESsoI2PbC0wHG1tcSVCKo1oqFemgPgaqrNTtfNBW3WjeGFqf3sw0PrroUIzJDpaBpO8y6vtGrh4pdzF3ZEBR7p3Oa4bjbgkMrbqxaf/jls5eJQiown6KV8YwBlXMGTg8FCpEKSChrmcqCX2KMTo03DBBFOVGhwRYKuccKAGZGMt9mhglRXQ4iP6+rhz39mucW/rN2Q3pqT7rS968/FYv8NJDCEG5kPIWaiRC9QVHIweeOe+s3Uyz9BMmTNCcW5Vjid3wLDKmU9njV8eOjLkljhM/zvrXnEN/Qed4KGIXrAf0HQPYqEZGDndZBhh2QK4pXtHqDCY4c2tQKLxcCC5kJ9wphIw8sJEJw4HJKkIhUj3J9EBQRiCnQCnNL4Y/cvBW86NpIOh3YgrmpErhWnLr6SVgeBQPU/FtjgBeyY6sdf+UayeAQawzrAI82QfnwB/88GBeUH4Abha0Zj8hAvB+oRwQ2d1BuomLkxPVURMW2srlfha/OUebtEuZjHlgsnqcHr/5bkKK6A3GsB0V2Dh0hkVV9K4awwczoq7SgUugisGf7XkBVr3ZoP4LN3CjDL5nhGKQAXXrhjgwayCHce+FAg1d8n0UhECVT/EWLij6bzuWj5HsBJuB4b7Dj2uDL65Wn4EFPSfSn+pDr+HdQBmwTjl4lOTgKXs4V9Wd+uUvHrkbAxNXLC3ylLHEtmLXxziWUwv/yXeQn9bwPfG1/HN08Tv2X7wNp4mvgji37Q4Ud6NvK8MyhmhN7i+GMp7MUYXblsFcmgmYYw9RCo+koGNVEySoguE49FoWQR0OcVF/rCkgrIQyj+yoiIGM585+NsSDhZ8R3Ti+reFNjwQ4ycATyDqMMSPrQqtHB4ijykMFN6pyAJ/ExzuzBksVKi2ehmyNAVUZkQZWJy8iYatXw99Y8yPR9TCEw8HZ1oIkVvP2KigzsB84WkhnSpZU/BgLhG8OGMbHQi5gZpIzwKtDfcddIinNferXNmJqJDh7LxIO53JjSrFFBo2g7szjQ1IsUBLGdeyc8NoC3+NXCZpVjr4miBXXostX4aa0JpiO5aH0HIwkcJTlAB4bSiDb5xkW+BBZUYGIoFywpIvwM/3cISpwhnPau5zvXe/D6NZ8Vl6HsZchex4unBKts8LVENANNQSdXGvKpEv4fWWkDjKIf5uwyt+zfSmdkNxY3qT7BXsXUZ7oYtCG+bdhWutdY4am+M3G2cghA0Dnmgdb7owcU701QQQhZ9IgGh4qVNwLlrlEDmfIQqvUIDwB9GQhYkcTgwvUQQQxkPL3nl4Wh9Cw1ErLy2AECYbD2LQACRTBCac1ehSZCR063iEiA4meGk1pZBApyYIQDDjAxllp40abXLFL/IC2DNOaT/cas846lKDQqrMhAqZEo/AJawby79QAvL0CaLFwcSyRMrM3QLSVsHNa0ZWlxsGlfkbedfcW1szdCQvfprWJM35lGD1vdbAaPNHaUF+k/uunpVTm+CEbe/2k7siLjReptE9ripAE+4NahdgZmp+tShKJgJ07Yosn/LILS7tl0WZmf5MYkLa6IlfoRtCAHsuFTXAXxL/Mf0vqmzXEtq3fBZDKikQa9pqWBjyHbxFeeD3FN+xr7ne09qGnwLqFTPXojxB5xjw/2nZi60ny+aN5ro8MofT1nbL4YrQEMyIKKiMFgpRtHHJUBOEk9CdihAXmSECFjzQfv/B/6eFL3LcV5RBp2+fnEkPVS45AOJx0vEeDR/+CUZGpNHsHshldTKMwbAD2OOyml8J/D/xLq+rVwm+i2exPPtYQUZz4jSR5gW8ANYaDVOmeEYcuxBKOPdX/Gva6wev0I1ZML5f14iuZ3SjgE1j68gnuUU8+VTRGyTPkYuF/33YLerh8y1XWfI2WxoScdYmAchnovrmNiy4L/OZPQj+HiEuplHMUvV/oUSyheJivKgLdJzyf7WZd7r/3T38Tvw4zNnwuJFtFnwmvNmwcfAz4XEj2yz4THizYePgZ8LjRrZZ8JnwZsPGwc+Ex41ss+Az4c2GjYOfCY8b2WbBZ8KbDRsHPxMeN7LNgs+ENxs2Dn4mPG5kmwWfCW82bBz8THjcyDYLPhPebNg4+JnwuJFtFnwmvNmwcfAz4XEj2yz4THizYePgZ8LjRrZZ8JnwZsPGwc+Ex41ss+Az4c2GjYOfCY8b2WbBZ8KbDRsHPxMeN7LNgs+ENxs2Dn4mPG5kmwWfCW82bBz8THjcyDYLPhPebNg4+JnwuJFtFnwmvNmwcfAz4XEj2yz4THizYePgZ8LjRrZZ8JnwZsPGwc+Ex41ss+Az4c2GjYOfCY8b2WbBZ8KbDRsHPxMeN7LNgs+ENxs2Dn4mPG5kmwWfCW82bBz8THjcyDYLPhPebNg4+JnwuJFtFnwmvNmwcfAz4XEj2yz4THizYePgZ8LjRrZZ8JnwZsPGwc+Ex41ss+Az4c2GjYOfCY8b2WbBZ8KbDRsHPxMeN7LNgs+ENxs2Dn4mPG5kmwWfCW82bBz8THjcyDYLPhPebNg4+JnwuJFtFnwmvNmwcfAz4XEj2yz4P6xN1v748sHXAAAAAElFTkSuQmCC",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=240x192>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated[0][2].to_PIL()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'signs'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(0.095, 0.085, 0.095, 0.4)  # best for 85M model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5153/5153 [23:11<00:00,  3.70it/s]\n"
     ]
    }
   ],
   "source": [
    "eva = evaluate_babi(generated, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(eva)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.11274985445371628, 'tesseract': 0.10304676887250146, 'union': 0.12090044634193674, 'readability': 0.21016883368911313}\n"
     ]
    }
   ],
   "source": [
    "print(eva)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'paddleocr': 0.12458761886279837, 'tesseract': 0.13137977876964874, 'union': 0.13758975354162623, 'readability': 0.6722297690665632}\n"
     ]
    }
   ],
   "source": [
    "print(eva)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pt2hfpy310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}