transformer_lang_understanding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "transformer_lang_understanding.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "16753f7c91bc4d90ace2425620d0fac4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0eb0abc530a144a59192b423568435ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1a0e7beea1884dc2a58b7f9a329101ec",
              "IPY_MODEL_03ae45a55ce04e81824e0932882b9f36",
              "IPY_MODEL_f9e5393ee97f420fa9f849d2935b4f00"
            ]
          }
        },
        "0eb0abc530a144a59192b423568435ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a0e7beea1884dc2a58b7f9a329101ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f127e2c4e3c7405f9a0df0fc86627dd0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Dl Completed...: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bd055b60db114f1b9c3aa1025a7b88da"
          }
        },
        "03ae45a55ce04e81824e0932882b9f36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_202082b5107245b4a5457e1174a1b846",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a81948fbff754d3fb622cec255cc8d3d"
          }
        },
        "f9e5393ee97f420fa9f849d2935b4f00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fac6d5b94c6744a5a5c6b25f08b33d6e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:08&lt;00:00,  3.63s/ url]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_04ab0104bff34b84b8de575beb339d59"
          }
        },
        "f127e2c4e3c7405f9a0df0fc86627dd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bd055b60db114f1b9c3aa1025a7b88da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "202082b5107245b4a5457e1174a1b846": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a81948fbff754d3fb622cec255cc8d3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fac6d5b94c6744a5a5c6b25f08b33d6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "04ab0104bff34b84b8de575beb339d59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f0d46832638469487cf137774f153a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ebf62ac1d6044678ad7af08ae9c44fe5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_53facc828a4543de82a7cd570e2f67c5",
              "IPY_MODEL_7c26b49696e44498bcae660237126f87",
              "IPY_MODEL_e245ee05ff8f400bb386f80bb87d1b31"
            ]
          }
        },
        "ebf62ac1d6044678ad7af08ae9c44fe5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53facc828a4543de82a7cd570e2f67c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8406ab80e7f14c238d2fd40d3e8bc53f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Dl Size...: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e04f804f7e943d3a4aa9c56b6c27d04"
          }
        },
        "7c26b49696e44498bcae660237126f87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_420e7b34ca99405488f33338efeddac2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c4198d5c7ab4845a00206e52d353efb"
          }
        },
        "e245ee05ff8f400bb386f80bb87d1b31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_367e44a03b444abdbd1699f0d284cd2c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 124/124 [00:08&lt;00:00, 44.32 MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f710353e960d4968a116def3f4039e0b"
          }
        },
        "8406ab80e7f14c238d2fd40d3e8bc53f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e04f804f7e943d3a4aa9c56b6c27d04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "420e7b34ca99405488f33338efeddac2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6c4198d5c7ab4845a00206e52d353efb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "367e44a03b444abdbd1699f0d284cd2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f710353e960d4968a116def3f4039e0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71c447ca718c4d128275a2457e2e6bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7597fae2bfcd4dafb1224b72826167f1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a7963673647c4b73a1d7a4bb384b2735",
              "IPY_MODEL_5dda6656eadc4386ba9e7b06ddb2ff95",
              "IPY_MODEL_62fbaf730d4f49d195b9387b947e5ccb"
            ]
          }
        },
        "7597fae2bfcd4dafb1224b72826167f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a7963673647c4b73a1d7a4bb384b2735": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ee332d02605e4177ac413f58c0fa8867",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Extraction completed...: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_992d062423ea42c18d38bcee07da5b9e"
          }
        },
        "5dda6656eadc4386ba9e7b06ddb2ff95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_912a751852a74059a15c71564d510c80",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08619a74f1ab4e9cbec22fa38a771f3f"
          }
        },
        "62fbaf730d4f49d195b9387b947e5ccb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0894695b49904f1386fb8a05b2f8cfe7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:08&lt;00:00,  8.69s/ file]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad016715bb234a0b8e998d4d08b78985"
          }
        },
        "ee332d02605e4177ac413f58c0fa8867": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "992d062423ea42c18d38bcee07da5b9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "912a751852a74059a15c71564d510c80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08619a74f1ab4e9cbec22fa38a771f3f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0894695b49904f1386fb8a05b2f8cfe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad016715bb234a0b8e998d4d08b78985": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "429f6c22f67f4328ac4d745f50c0bc3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5830b6d25aeb41b59d91a54c71f30e55",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_52ef71873bb3450091127825dfb7dfea",
              "IPY_MODEL_8b2b548522af4435adc5f9d0c2a4ff38",
              "IPY_MODEL_3557f61bb7e84febba0a9d003827d3e4"
            ]
          }
        },
        "5830b6d25aeb41b59d91a54c71f30e55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52ef71873bb3450091127825dfb7dfea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7718c8cbc6854e29a73ca98eb1114728",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8b36d30b558e4bc8b0ef891037adf21d"
          }
        },
        "8b2b548522af4435adc5f9d0c2a4ff38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_799c176f26cd45fc9e1bc12bcce3f979",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f484d3d3637f489ab6486561054ae36c"
          }
        },
        "3557f61bb7e84febba0a9d003827d3e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ae46ab11b4f4382bdfda0f86e754742",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51655/0 [00:13&lt;00:00, 3861.37 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fa219f8d342e4a1d9e74f9fbefdba5ad"
          }
        },
        "7718c8cbc6854e29a73ca98eb1114728": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8b36d30b558e4bc8b0ef891037adf21d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "799c176f26cd45fc9e1bc12bcce3f979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f484d3d3637f489ab6486561054ae36c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ae46ab11b4f4382bdfda0f86e754742": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fa219f8d342e4a1d9e74f9fbefdba5ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cead874275994f86882ef02eb28f2bc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_84fcfb9d40a94bd08fef7cd1dbb0ea35",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9981fa8e04834767b947082257d5a5c9",
              "IPY_MODEL_d77a52da141d43218f89236628cc1875",
              "IPY_MODEL_a126f5bd85604323916f5e498fdaf372"
            ]
          }
        },
        "84fcfb9d40a94bd08fef7cd1dbb0ea35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9981fa8e04834767b947082257d5a5c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0fc6b972c9a44267a12f5b2f7e9a2b3e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1fc17b7eca9946cb9153f7e24efe7dfe"
          }
        },
        "d77a52da141d43218f89236628cc1875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6af81bac5f4c4d05ac776a425114e4d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 51785,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 51784,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ec12c9cc4a5140ec9ebd2d606911a263"
          }
        },
        "a126f5bd85604323916f5e498fdaf372": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a13dd4c7a43348dea56354eb0b7312aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51784/51785 [00:00&lt;00:00, 188397.61 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f285221ed214e96a2efc1d7220ea031"
          }
        },
        "0fc6b972c9a44267a12f5b2f7e9a2b3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1fc17b7eca9946cb9153f7e24efe7dfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6af81bac5f4c4d05ac776a425114e4d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ec12c9cc4a5140ec9ebd2d606911a263": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a13dd4c7a43348dea56354eb0b7312aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f285221ed214e96a2efc1d7220ea031": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e920d6a69aa246cda03df23f89ac7ca3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_530d1a2d91084c5eb0bba93ef9858735",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8bbb821a64884e6c9e499a2ea02bf207",
              "IPY_MODEL_cb77049f59654e6b9af169c4e0dfee66",
              "IPY_MODEL_c49c7678f4404a8a97088422f79e926e"
            ]
          }
        },
        "530d1a2d91084c5eb0bba93ef9858735": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bbb821a64884e6c9e499a2ea02bf207": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3b1a21269c2645a886053fde3888c5d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_256cc0074c794a998fb3043bf24b8bf5"
          }
        },
        "cb77049f59654e6b9af169c4e0dfee66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_321f8aef43874426a01aa5ba1cf9721f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_824d315c8a1746c7b40eb7af5bde9aff"
          }
        },
        "c49c7678f4404a8a97088422f79e926e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6384d5cfdbc04efe8035db7aaf088030",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1163/0 [00:00&lt;00:00, 3280.15 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35847d8724b64243a863bf7fc22a7469"
          }
        },
        "3b1a21269c2645a886053fde3888c5d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "256cc0074c794a998fb3043bf24b8bf5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "321f8aef43874426a01aa5ba1cf9721f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "824d315c8a1746c7b40eb7af5bde9aff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6384d5cfdbc04efe8035db7aaf088030": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35847d8724b64243a863bf7fc22a7469": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "849984e37d8c4b749f1ed96b24c76011": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c64d1ae72eda45cf8af4210a986908fd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3bfa07b9c5e04bf3a9d89205aed60a13",
              "IPY_MODEL_b5a22b59b0e9417187048037d3b67a59",
              "IPY_MODEL_acfa6331e6664a019d14fb3789b36275"
            ]
          }
        },
        "c64d1ae72eda45cf8af4210a986908fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3bfa07b9c5e04bf3a9d89205aed60a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4fed5023e584452ab47b7370534a945e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_77af4e1deb40447aa17b2353d899abc4"
          }
        },
        "b5a22b59b0e9417187048037d3b67a59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2b6237a5187d4859a27988b40e166657",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1193,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1192,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9407ce9a208e4762be9de3d1dbd26362"
          }
        },
        "acfa6331e6664a019d14fb3789b36275": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_45d5e501cff64d12824c8bef6b2e677c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1192/1193 [00:00&lt;00:00, 20371.15 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c343597483ab41daad271c28aefcadc6"
          }
        },
        "4fed5023e584452ab47b7370534a945e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "77af4e1deb40447aa17b2353d899abc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b6237a5187d4859a27988b40e166657": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9407ce9a208e4762be9de3d1dbd26362": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "45d5e501cff64d12824c8bef6b2e677c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c343597483ab41daad271c28aefcadc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5a5cce35e6e49f3bfd22307b26d060e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c9133535861249fca461dfd590c49baa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5462d0d179c44e5c83f9938d49a41a35",
              "IPY_MODEL_d49cb112860c4cb599f1973aadb46ac4",
              "IPY_MODEL_dbec97768af84913b94ceb45adb3e2ef"
            ]
          }
        },
        "c9133535861249fca461dfd590c49baa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5462d0d179c44e5c83f9938d49a41a35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_205309a901614f8980eddba44adac677",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f8270e64ad74e449c9380d57c4e8cc9"
          }
        },
        "d49cb112860c4cb599f1973aadb46ac4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6ace170906354fc192d4be373f8eb371",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16e85358dca04486944d17492f856ab2"
          }
        },
        "dbec97768af84913b94ceb45adb3e2ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e67aa0aa7c4242dcb28bca6ad402459f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1480/0 [00:00&lt;00:00, 3452.09 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dbe41669eb714a8cacca3389f958dd87"
          }
        },
        "205309a901614f8980eddba44adac677": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f8270e64ad74e449c9380d57c4e8cc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ace170906354fc192d4be373f8eb371": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "16e85358dca04486944d17492f856ab2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e67aa0aa7c4242dcb28bca6ad402459f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dbe41669eb714a8cacca3389f958dd87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "96ad130961814db69a9cb6bba3f2d6e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_589302f494b14772818f0b12210a2f6e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbdc42c0c11143bfbd3e950d4fe72e98",
              "IPY_MODEL_7e80f8b735d8444a97e748d38fd1e590",
              "IPY_MODEL_27be4eda63344846b5e678079f5f2517"
            ]
          }
        },
        "589302f494b14772818f0b12210a2f6e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbdc42c0c11143bfbd3e950d4fe72e98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eb64344859fe4f24aeaa6aaffc1b4e5a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f246a090246406f9cd48f29068cce5e"
          }
        },
        "7e80f8b735d8444a97e748d38fd1e590": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_556e402547224d96b2bf9e9e4528531a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1803,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1802,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_255737f3593a47a0ae1f3514d0054f4c"
          }
        },
        "27be4eda63344846b5e678079f5f2517": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_65701b601c7c4eceb328ba95a0b22cd5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1802/1803 [00:00&lt;00:00, 29695.88 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_946cab234b25417db196ac98c0394d9d"
          }
        },
        "eb64344859fe4f24aeaa6aaffc1b4e5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f246a090246406f9cd48f29068cce5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "556e402547224d96b2bf9e9e4528531a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "255737f3593a47a0ae1f3514d0054f4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65701b601c7c4eceb328ba95a0b22cd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "946cab234b25417db196ac98c0394d9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qg3CdrjWloGA"
      },
      "source": [
        "# Transformer Model for Language Understanding\n",
        "\n",
        "We will build a Transformer model to translate Portugese to English.\n",
        "\n",
        "The core idea behind Transformer models is *self-attention* - the ability to attend to different positions of the input sequence to compute a representation of that sequence. Transformer models create stacks of self-attention layers instead of RNNs or CNNs.\n",
        "\n",
        "This approach has the following advantages:\n",
        "- No assumptions about the temporal/spatial relationships of the data are required\n",
        "- Layer outputs can be calculated in parallel\n",
        "- Distant items can affect each other's inputs without passing through many RNN steps or convolution layers.\n",
        "- It can learn long-range dependencies\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ia96nWKfpHKr"
      },
      "source": [
        "### Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OlDVqfaRlhzo",
        "outputId": "5feaedb6-3278-40da-e26c-f74aacd07939"
      },
      "source": [
        "!pip install tensorflow_datasets\n",
        "!pip install -U tensorflow-text"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tensorflow_datasets in /usr/local/lib/python3.7/dist-packages (4.0.1)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (3.17.3)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.15.0)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.1.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (4.62.0)\n",
            "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (21.2.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.19.5)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.3.4)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.16.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (5.2.2)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.2.0)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (2.3)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.12.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow_datasets) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow_datasets) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow_datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow_datasets) (3.0.4)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->tensorflow_datasets) (3.5.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-metadata->tensorflow_datasets) (1.53.0)\n",
            "Collecting tensorflow-text\n",
            "  Downloading tensorflow_text-2.6.0-cp37-cp37m-manylinux1_x86_64.whl (4.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow<2.7,>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text) (2.6.0)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text) (0.12.0)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (0.4.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (0.2.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.1.2)\n",
            "Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (2.6.0)\n",
            "Requirement already satisfied: tensorflow-estimator~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (2.6.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (0.37.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.19.5)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.6.3)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (3.7.4.3)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.12)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (3.17.3)\n",
            "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (2.6.0)\n",
            "Requirement already satisfied: clang~=5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (5.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.1.0)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (3.1.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (0.12.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (3.3.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.39.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.12.1)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.7,>=2.6.0->tensorflow-text) (1.15.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.5.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (0.4.5)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (57.4.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.34.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.8.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (0.6.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (3.3.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (4.6.4)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (0.4.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow-text) (3.5.0)\n",
            "Installing collected packages: tensorflow-text\n",
            "Successfully installed tensorflow-text-2.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w5W8utr4pKYg"
      },
      "source": [
        "import collections\n",
        "import logging\n",
        "import os\n",
        "import pathlib\n",
        "import re\n",
        "import string\n",
        "import sys\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import tensorflow_datasets as tfds\n",
        "import tensorflow_text as text\n",
        "import tensorflow as tf"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gnLhElnOpL5r"
      },
      "source": [
        "logging.getLogger('tensorflow').setLevel(logging.ERROR)  # suppress warnings"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "koaTYkoWpY3d"
      },
      "source": [
        "### Dataset\n",
        "\n",
        "We will be using the [Portuguese-English translation dataset](https://github.com/neulab/word-embeddings-for-nmt) from the [TED Talks Open Translation Project](https://www.ted.com/participate/translate). The dataset contains about 50000 training examples, 1100 validation examples and 2000 test examples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367,
          "referenced_widgets": [
            "16753f7c91bc4d90ace2425620d0fac4",
            "0eb0abc530a144a59192b423568435ac",
            "1a0e7beea1884dc2a58b7f9a329101ec",
            "03ae45a55ce04e81824e0932882b9f36",
            "f9e5393ee97f420fa9f849d2935b4f00",
            "f127e2c4e3c7405f9a0df0fc86627dd0",
            "bd055b60db114f1b9c3aa1025a7b88da",
            "202082b5107245b4a5457e1174a1b846",
            "a81948fbff754d3fb622cec255cc8d3d",
            "fac6d5b94c6744a5a5c6b25f08b33d6e",
            "04ab0104bff34b84b8de575beb339d59",
            "0f0d46832638469487cf137774f153a2",
            "ebf62ac1d6044678ad7af08ae9c44fe5",
            "53facc828a4543de82a7cd570e2f67c5",
            "7c26b49696e44498bcae660237126f87",
            "e245ee05ff8f400bb386f80bb87d1b31",
            "8406ab80e7f14c238d2fd40d3e8bc53f",
            "0e04f804f7e943d3a4aa9c56b6c27d04",
            "420e7b34ca99405488f33338efeddac2",
            "6c4198d5c7ab4845a00206e52d353efb",
            "367e44a03b444abdbd1699f0d284cd2c",
            "f710353e960d4968a116def3f4039e0b",
            "71c447ca718c4d128275a2457e2e6bef",
            "7597fae2bfcd4dafb1224b72826167f1",
            "a7963673647c4b73a1d7a4bb384b2735",
            "5dda6656eadc4386ba9e7b06ddb2ff95",
            "62fbaf730d4f49d195b9387b947e5ccb",
            "ee332d02605e4177ac413f58c0fa8867",
            "992d062423ea42c18d38bcee07da5b9e",
            "912a751852a74059a15c71564d510c80",
            "08619a74f1ab4e9cbec22fa38a771f3f",
            "0894695b49904f1386fb8a05b2f8cfe7",
            "ad016715bb234a0b8e998d4d08b78985",
            "429f6c22f67f4328ac4d745f50c0bc3c",
            "5830b6d25aeb41b59d91a54c71f30e55",
            "52ef71873bb3450091127825dfb7dfea",
            "8b2b548522af4435adc5f9d0c2a4ff38",
            "3557f61bb7e84febba0a9d003827d3e4",
            "7718c8cbc6854e29a73ca98eb1114728",
            "8b36d30b558e4bc8b0ef891037adf21d",
            "799c176f26cd45fc9e1bc12bcce3f979",
            "f484d3d3637f489ab6486561054ae36c",
            "0ae46ab11b4f4382bdfda0f86e754742",
            "fa219f8d342e4a1d9e74f9fbefdba5ad",
            "cead874275994f86882ef02eb28f2bc9",
            "84fcfb9d40a94bd08fef7cd1dbb0ea35",
            "9981fa8e04834767b947082257d5a5c9",
            "d77a52da141d43218f89236628cc1875",
            "a126f5bd85604323916f5e498fdaf372",
            "0fc6b972c9a44267a12f5b2f7e9a2b3e",
            "1fc17b7eca9946cb9153f7e24efe7dfe",
            "6af81bac5f4c4d05ac776a425114e4d1",
            "ec12c9cc4a5140ec9ebd2d606911a263",
            "a13dd4c7a43348dea56354eb0b7312aa",
            "9f285221ed214e96a2efc1d7220ea031",
            "e920d6a69aa246cda03df23f89ac7ca3",
            "530d1a2d91084c5eb0bba93ef9858735",
            "8bbb821a64884e6c9e499a2ea02bf207",
            "cb77049f59654e6b9af169c4e0dfee66",
            "c49c7678f4404a8a97088422f79e926e",
            "3b1a21269c2645a886053fde3888c5d6",
            "256cc0074c794a998fb3043bf24b8bf5",
            "321f8aef43874426a01aa5ba1cf9721f",
            "824d315c8a1746c7b40eb7af5bde9aff",
            "6384d5cfdbc04efe8035db7aaf088030",
            "35847d8724b64243a863bf7fc22a7469",
            "849984e37d8c4b749f1ed96b24c76011",
            "c64d1ae72eda45cf8af4210a986908fd",
            "3bfa07b9c5e04bf3a9d89205aed60a13",
            "b5a22b59b0e9417187048037d3b67a59",
            "acfa6331e6664a019d14fb3789b36275",
            "4fed5023e584452ab47b7370534a945e",
            "77af4e1deb40447aa17b2353d899abc4",
            "2b6237a5187d4859a27988b40e166657",
            "9407ce9a208e4762be9de3d1dbd26362",
            "45d5e501cff64d12824c8bef6b2e677c",
            "c343597483ab41daad271c28aefcadc6",
            "e5a5cce35e6e49f3bfd22307b26d060e",
            "c9133535861249fca461dfd590c49baa",
            "5462d0d179c44e5c83f9938d49a41a35",
            "d49cb112860c4cb599f1973aadb46ac4",
            "dbec97768af84913b94ceb45adb3e2ef",
            "205309a901614f8980eddba44adac677",
            "6f8270e64ad74e449c9380d57c4e8cc9",
            "6ace170906354fc192d4be373f8eb371",
            "16e85358dca04486944d17492f856ab2",
            "e67aa0aa7c4242dcb28bca6ad402459f",
            "dbe41669eb714a8cacca3389f958dd87",
            "96ad130961814db69a9cb6bba3f2d6e9",
            "589302f494b14772818f0b12210a2f6e",
            "bbdc42c0c11143bfbd3e950d4fe72e98",
            "7e80f8b735d8444a97e748d38fd1e590",
            "27be4eda63344846b5e678079f5f2517",
            "eb64344859fe4f24aeaa6aaffc1b4e5a",
            "6f246a090246406f9cd48f29068cce5e",
            "556e402547224d96b2bf9e9e4528531a",
            "255737f3593a47a0ae1f3514d0054f4c",
            "65701b601c7c4eceb328ba95a0b22cd5",
            "946cab234b25417db196ac98c0394d9d"
          ]
        },
        "id": "-CzLJmLppyIX",
        "outputId": "b0b7e19a-1ca7-488c-b641-fc9786164cfe"
      },
      "source": [
        "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n",
        "                               as_supervised=True)\n",
        "train_examples, val_examples = examples['train'], examples['validation']"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mDownloading and preparing dataset ted_hrlr_translate/pt_to_en/1.0.0 (download: 124.94 MiB, generated: Unknown size, total: 124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\u001b[0m\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "16753f7c91bc4d90ace2425620d0fac4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Dl Completed...: 0 url [00:00, ? url/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0f0d46832638469487cf137774f153a2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Dl Size...: 0 MiB [00:00, ? MiB/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71c447ca718c4d128275a2457e2e6bef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Extraction completed...: 0 file [00:00, ? file/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "429f6c22f67f4328ac4d745f50c0bc3c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompletePRO7WE/ted_hrlr_translate-train.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cead874275994f86882ef02eb28f2bc9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/51785 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e920d6a69aa246cda03df23f89ac7ca3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompletePRO7WE/ted_hrlr_translate-validation.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "849984e37d8c4b749f1ed96b24c76011",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1193 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e5a5cce35e6e49f3bfd22307b26d060e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompletePRO7WE/ted_hrlr_translate-test.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "96ad130961814db69a9cb6bba3f2d6e9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1803 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mDataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R7WyRN1xp-No"
      },
      "source": [
        "### Text Tokenization and Detokenization\n",
        "\n",
        "We use the pretrained subword tokenizer that uses an BERT tokenizer optimized for this dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "id": "Yrd-h2nRqi2y",
        "outputId": "5a0e0297-a8c8-4995-8a58-6dcad7b7f409"
      },
      "source": [
        "model_name = \"ted_hrlr_translate_pt_en_converter\"\n",
        "tf.keras.utils.get_file(\n",
        "    f\"{model_name}.zip\",\n",
        "    f\"https://storage.googleapis.com/download.tensorflow.org/models/{model_name}.zip\",\n",
        "    cache_dir='.', cache_subdir='', extract=True\n",
        ")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/download.tensorflow.org/models/ted_hrlr_translate_pt_en_converter.zip\n",
            "188416/184801 [==============================] - 0s 0us/step\n",
            "196608/184801 [===============================] - 0s 0us/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'./ted_hrlr_translate_pt_en_converter.zip'"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fbj8nHovr7uk"
      },
      "source": [
        "#tokenizers contains both the english and portugese tokenizer,\n",
        "# with tokenize, detokenize and lookup methods\n",
        "tokenizers = tf.saved_model.load(model_name)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GCODcwwir5Hb"
      },
      "source": [
        "### Input Pipeline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pfVEnFuUts62"
      },
      "source": [
        "Helper function to encode batches of raw text:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LHEe_kVTt0aa"
      },
      "source": [
        "def tokenize_pairs(pt, en):\n",
        "    pt = tokenizers.pt.tokenize(pt)\n",
        "    # Convert from ragged to dense, padding with zeros.\n",
        "    pt = pt.to_tensor()\n",
        "    en = tokenizers.en.tokenize(en)\n",
        "    # Convert from ragged to dense, padding with zeros.\n",
        "    en = en.to_tensor()\n",
        "    return pt, en"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jY92gvDduKdt"
      },
      "source": [
        "#constants\n",
        "BUFFER_SIZE = 20000\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPZ3dRHQuZml"
      },
      "source": [
        "Helper function to preprocess batches. Then preprocess data:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqX3Zm6EuXiz"
      },
      "source": [
        "def make_batches(ds):\n",
        "  return (\n",
        "      ds\n",
        "      .cache()\n",
        "      .shuffle(BUFFER_SIZE)\n",
        "      .batch(BATCH_SIZE)\n",
        "      .map(tokenize_pairs, num_parallel_calls=tf.data.AUTOTUNE)\n",
        "      .prefetch(tf.data.AUTOTUNE))\n",
        "  \n",
        "train_batches = make_batches(train_examples)\n",
        "val_batches = make_batches(val_examples)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACzzLYMVuo9D"
      },
      "source": [
        "### Positional Encoding\n",
        "\n",
        "Since attention layers see thier input as a set of vectors with no sequential order, we need a positional encoding to give the model information about relative positioning of the tokens.\n",
        "\n",
        "We implement the positional encoding as an embedding vector. Embeddings represent a token in d-dimensional space, so tokens with similar meaning will be closer to each other, but embeddings do not encode relative positioning of tokens in a sentence. After adding positional encodings, we can enhance our groupings/distance between tokens basd on similarity and sentence positioning (in d-dimensional space)\n",
        "\n",
        "We calculate positional encodings by the following formulas:\n",
        "$${PE_{(pos, 2i)} = \\sin(pos / 10000^{2i / d_{model}})} $$\n",
        "$${PE_{(pos, 2i+1)} = \\cos(pos / 10000^{2i / d_{model}})} $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OitjHuQ2v_py"
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc9sILUHwBPC"
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "  return tf.cast(pos_encoding, dtype=tf.float32)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ZqdiN2KwiEn"
      },
      "source": [
        "### Masking\n",
        "\n",
        "Mask the pad tokens in the batch of sequences, to ensure that the model does not treat padding as input (mask outputs 1 where pad value is 0 in sequence, else output 0)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K-QSdheBwy_t"
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  # add extra dimensions to add the padding\n",
        "  # to the attention logits.\n",
        "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nG1feNDpxjJt"
      },
      "source": [
        "We will also create a look-ahead mask that masks future tokens in a sequence (ie indicates which entries should not be used). So to predict the nth token, we only use the first $n-1$ tokens."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GqWxIUFvyJK7"
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "  mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "  return mask  # (seq_len, seq_len)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rOnQDpggyVIJ"
      },
      "source": [
        "### Scaled Dot-Product Attention\n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/scaled_attention.png\" width=\"500\" alt=\"scaled_dot_product_attention\">\n",
        "\n",
        "We use the attention function defined by\n",
        "$${Attention(Q, K, V) = softmax_k\\left(\\frac{QK^T}{\\sqrt{d_k}}\\right) V} $$\n",
        "Where we have inputs Q (query), K (key), V (value).\n",
        "\n",
        "We scale the dot product by the square root of the depth to balance out the growth of the dot product for large magnitude vectors, which would result in small gradients on the softmax function, slowing optimization.\n",
        "\n",
        "The mask is multiplied by -1e9 since the mask is summed with the scaled matrix multiplication of Q and K before the softmax. We want to zero out these cells, and hence aim for large negative inputs to our softmax.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pBCWTtue4PDx"
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "  \"\"\"Calculate the attention weights\n",
        "    q: query shape == (..., seq_len_q, depth)\n",
        "    k: key shape == (..., seq_len_k, depth)\n",
        "    v: value shape == (..., seq_len_v, depth_v)\n",
        "    mask: Float tensor with shape broadcastable\n",
        "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "\n",
        "    Return: output, attention_weights\n",
        "  \"\"\"\n",
        "\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "  # add the mask to the scaled tensor.\n",
        "  if mask is not None:\n",
        "    scaled_attention_logits += (mask * -1e9)\n",
        "\n",
        "  # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "  # add up to 1.\n",
        "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n",
        "\n",
        "  output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PzozmuukFv90"
      },
      "source": [
        "The values after the softmax normalization on K determine the amount of importance to give to Q.\n",
        "\n",
        "The output represents the multiplication of attention weights and the V (value) vector, ensuring that tokens we want to focus on are kepy while irrelevant ones are flushed out.\n",
        "\n",
        "Examples:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06o4Uo4d5I0_"
      },
      "source": [
        "#helper function to demonstrate attention weight and output on multiplication\n",
        "def print_out(q, k, v):\n",
        "  temp_out, temp_attn = scaled_dot_product_attention(\n",
        "      q, k, v, None)\n",
        "  print('Attention weights are:')\n",
        "  print(temp_attn)\n",
        "  print('Output is:')\n",
        "  print(temp_out)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8CjaDUlyGWqF"
      },
      "source": [
        "np.set_printoptions(suppress=True)\n",
        "\n",
        "temp_k = tf.constant([[10, 0, 0],\n",
        "                      [0, 10, 0],\n",
        "                      [0, 0, 10],\n",
        "                      [0, 0, 10]], dtype=tf.float32)  # (4, 3)\n",
        "\n",
        "temp_v = tf.constant([[1, 0],\n",
        "                      [10, 0],\n",
        "                      [100, 5],\n",
        "                      [1000, 6]], dtype=tf.float32)  # (4, 2)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w7z4tMr4GsKx",
        "outputId": "4cc94fa7-cb5f-491c-b0df-810525609f17"
      },
      "source": [
        "# This `query` aligns with the second `key`,\n",
        "# so the second `value` is returned.\n",
        "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)\n",
        "print()\n",
        "\n",
        "# This query aligns with a repeated key (third and fourth),\n",
        "# so all associated values get averaged.\n",
        "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)\n",
        "print()\n",
        "\n",
        "# This query aligns equally with the first and second key,\n",
        "# so their values get averaged.\n",
        "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n",
            "\n",
            "Attention weights are:\n",
            "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n",
            "\n",
            "Attention weights are:\n",
            "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gvCKgKL1H9NE"
      },
      "source": [
        "Since Q is a matrix, we can combine multiple queries in 1 shot:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rnGLn0BUHJB1",
        "outputId": "fa840224-3a1f-4f9b-cc42-09147f37a5b8"
      },
      "source": [
        "temp_q = tf.constant([[0, 0, 10],\n",
        "                      [0, 10, 0],\n",
        "                      [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor(\n",
            "[[0.  0.  0.5 0.5]\n",
            " [0.  1.  0.  0. ]\n",
            " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor(\n",
            "[[550.    5.5]\n",
            " [ 10.    0. ]\n",
            " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbMzhcrKIDg8"
      },
      "source": [
        "### Multi-Head Attention\n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\" width=\"500\" alt=\"multi-head attention\">\n",
        "\n",
        "\n",
        "Multi-head attention consists of four parts:\n",
        "*    Linear layers.\n",
        "*    Scaled dot-product attention.\n",
        "*    Final linear layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DxeYCevxIKbS"
      },
      "source": [
        "Each multi-head attention gets inputs Q (query), K (key), V (value), which are put through dense linear layers before the multi-head attention function.\n",
        "\n",
        "(K,Q,V) are passed through separate linear layers for each attention head.\n",
        "For efficiency/simplicity, we implement this using a single dense layer with `num_heads` times as many outputs and rearrange the output to shape `(batch, num_heads, ...)` before the attention function.\n",
        "\n",
        "We apply the `scaled_dot_product_attention` function to a single cell, then broadcast it for efficiency. Note an appropriate mask must be used in the attention step. After, the attention output is concatenated and put through a final dense layer.\n",
        "\n",
        "We use multiple. attention heads because it allows the model to jointly attend to information from difference representation subspaces at different positions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NtYZ9N8vKaYb"
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "\n",
        "    assert d_model % self.num_heads == 0\n",
        "\n",
        "    self.depth = d_model // self.num_heads\n",
        "\n",
        "    self.wq = tf.keras.layers.Dense(d_model)\n",
        "    self.wk = tf.keras.layers.Dense(d_model)\n",
        "    self.wv = tf.keras.layers.Dense(d_model)\n",
        "\n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "\n",
        "  def split_heads(self, x, batch_size):\n",
        "    \"\"\"Split the last dimension into (num_heads, depth).\n",
        "    Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "    \"\"\"\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "\n",
        "  def call(self, v, k, q, mask):\n",
        "    batch_size = tf.shape(q)[0]\n",
        "\n",
        "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
        "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
        "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
        "\n",
        "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "\n",
        "    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
        "        q, k, v, mask)\n",
        "\n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "    concat_attention = tf.reshape(scaled_attention,\n",
        "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    return output, attention_weights"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C_HWfh8cMWFB"
      },
      "source": [
        "### Point Wise Feed Forward Network\n",
        "\n",
        "Point wise feed forward netowrk consists of 2 fully-connected layers with ReLU activations in between."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8chfdKluMmwL"
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n",
        "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n",
        "  ])"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9xcB72VAMrvG"
      },
      "source": [
        "### Encoder and Decoder\n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/transformer.png\" width=\"600\" alt=\"transformer\">"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MCeRwOeJMwvj"
      },
      "source": [
        "The transformer model follows the same general pattern as a standard sequence to sequence with attention model;\n",
        "- The input sentence is passed through N encoder layers that generate an output for each token in the sequence\n",
        "- The decoder attends to the encoder's output and its own input (self-attention) to predict the next word"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OGUTNXShNLvL"
      },
      "source": [
        "#### Encoder Layer\n",
        "\n",
        "Each encoder layer consists of sublayers:\n",
        "- Multi-head attention (with padding mask)\n",
        "- Point-wise feed forward networks\n",
        "\n",
        "Each of the sublayers has a residual connection (to avoid vanishing gradients) followed by layer normalization.\n",
        "\n",
        "For each of the N encoder layers in the transformer, the output of each sublayer is `LayerNorm(x + Sublayer(x))`, and normalization is done on the `d_model` (last) axis."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UnyGCB3JOCAt"
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(EncoderLayer, self).__init__()\n",
        "\n",
        "    self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "    attn_output = self.dropout1(attn_output, training=training)\n",
        "    out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
        "    ffn_output = self.dropout2(ffn_output, training=training)\n",
        "    out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    return out2"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZqZL43ogOlr5"
      },
      "source": [
        "#### Decoder Layer\n",
        "\n",
        "Each of the N decoder layer consists of sublayers:\n",
        "- Masked multi-head attention (with look ahead mask and padding mask)\n",
        "- Multi-head attention (with padding mask). V (value) and K (key) receive the *encoder output* as inputs. Q (query) receives the *output from the masked multi-head attention sublayer.*\n",
        "- Point wise feed forward networks\n",
        "\n",
        "As Q receives the output from the decoder's first attention block, and K receives the encoder output, the attention weights represent the importance given to the decoder's input based on the encoder's output; the decoder predicts teh next token by looking at the encoder output and self-attending its own output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQQMn1McP8i8"
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(DecoderLayer, self).__init__()\n",
        "\n",
        "    self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "    self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "  def call(self, x, enc_output, training,\n",
        "           look_ahead_mask, padding_mask):\n",
        "    # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn1 = self.dropout1(attn1, training=training)\n",
        "    out1 = self.layernorm1(attn1 + x)\n",
        "\n",
        "    attn2, attn_weights_block2 = self.mha2(\n",
        "        enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn2 = self.dropout2(attn2, training=training)\n",
        "    out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
        "\n",
        "    ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
        "    ffn_output = self.dropout3(ffn_output, training=training)\n",
        "    out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
        "\n",
        "    return out3, attn_weights_block1, attn_weights_block2"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VyaTjmXGQt5S"
      },
      "source": [
        "#### Encoder\n",
        "\n",
        "The `Encoder` consists of:\n",
        "1.   Input Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N encoder layers\n",
        "\n",
        "The input is first put through an embedded which is summed with the positional encoding. The sum is then inputted to the encoding layers. The output of the encoder is then the input to the decoder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STKPv530RCyu"
      },
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Encoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "\n",
        "    self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding,\n",
        "                                            self.d_model)\n",
        "\n",
        "    self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)\n",
        "                       for _ in range(num_layers)]\n",
        "\n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "\n",
        "    # adding embedding and position encoding.\n",
        "    x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "\n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    for i in range(self.num_layers):\n",
        "      x = self.enc_layers[i](x, training, mask)\n",
        "\n",
        "    return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aBJZ7NMgRoBf"
      },
      "source": [
        "#### Decoder\n",
        "\n",
        "The `Decoder` consists of:\n",
        "1.   Output Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N decoder layers\n",
        "\n",
        "The target is put through an embedding which is summed with the positional encoding. This sum is the input to the decoder layers. The output of the decoder is then the input to the final linear layer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FopDzIYgSF8t"
      },
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Decoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "\n",
        "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "\n",
        "    self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate)\n",
        "                       for _ in range(num_layers)]\n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "  def call(self, x, enc_output, training,\n",
        "           look_ahead_mask, padding_mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    attention_weights = {}\n",
        "\n",
        "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "\n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    for i in range(self.num_layers):\n",
        "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
        "                                             look_ahead_mask, padding_mask)\n",
        "\n",
        "      attention_weights[f'decoder_layer{i+1}_block1'] = block1\n",
        "      attention_weights[f'decoder_layer{i+1}_block2'] = block2\n",
        "\n",
        "    # x.shape == (batch_size, target_seq_len, d_model)\n",
        "    return x, attention_weights"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idUsZ79gTDZA"
      },
      "source": [
        "### Create the Transformer\n",
        "\n",
        "The transformer implements the encoder, decoder and a final linear layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Af7VMAywTJmQ"
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               target_vocab_size, pe_input, pe_target, rate=0.1):\n",
        "    super().__init__()\n",
        "    self.encoder = Encoder(num_layers, d_model, num_heads, dff,\n",
        "                             input_vocab_size, pe_input, rate)\n",
        "\n",
        "    self.decoder = Decoder(num_layers, d_model, num_heads, dff,\n",
        "                           target_vocab_size, pe_target, rate)\n",
        "\n",
        "    self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "\n",
        "  def call(self, inputs, training):\n",
        "    # Keras models prefer if you pass all your inputs in the first argument\n",
        "    inp, tar = inputs\n",
        "\n",
        "    enc_padding_mask, look_ahead_mask, dec_padding_mask = self.create_masks(inp, tar)\n",
        "\n",
        "    enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n",
        "\n",
        "    # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "    dec_output, attention_weights = self.decoder(\n",
        "        tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
        "\n",
        "    final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
        "\n",
        "    return final_output, attention_weights\n",
        "\n",
        "  def create_masks(self, inp, tar):\n",
        "    # Encoder padding mask\n",
        "    enc_padding_mask = create_padding_mask(inp)\n",
        "\n",
        "    # Used in the 2nd attention block in the decoder.\n",
        "    # This padding mask is used to mask the encoder outputs.\n",
        "    dec_padding_mask = create_padding_mask(inp)\n",
        "\n",
        "    # Used in the 1st attention block in the decoder.\n",
        "    # It is used to pad and mask future tokens in the input received by\n",
        "    # the decoder.\n",
        "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
        "    dec_target_padding_mask = create_padding_mask(tar)\n",
        "    look_ahead_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
        "\n",
        "    return enc_padding_mask, look_ahead_mask, dec_padding_mask"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZcNPfdrG0o_A"
      },
      "source": [
        "### Hyperparameters\n",
        "\n",
        "We use a smaller number for `num_layers, d_model, dff` for sake of computational efficiency. In the paper [Attention is all you Need](https://arxiv.org/abs/1706.03762) the hyperparams used are `num_layers=6, d_model=512, dff=2048`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wKguPAxL1J5F"
      },
      "source": [
        "num_layers = 4\n",
        "d_model = 128\n",
        "dff = 512\n",
        "num_heads = 8\n",
        "dropout_rate = 0.1"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8gYWYCJv1W54"
      },
      "source": [
        "### Optimizer\n",
        "\n",
        "We use Adam optimizer with custom learning rate scheduler according to formula given by the [Attention is all you Need](https://arxiv.org/abs/1706.03762) paper\n",
        "\n",
        "$${lrate = d_{model}^{-0.5} * \\min(step{\\_}num^{-0.5}, step{\\_}num \\cdot warmup{\\_}steps^{-1.5})}$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKoXBhtd2B_k"
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "\n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "\n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
        "\n",
        "#create the learning rate scheduler and optimizer\n",
        "learning_rate = CustomSchedule(d_model)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,\n",
        "                                     epsilon=1e-9)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kK0LfD_J2jqk"
      },
      "source": [
        "### Loss and Metrics\n",
        "\n",
        "The target sequences are padded, so it's important to apply a padding mask when calculating the loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqjmP_qt2wIj"
      },
      "source": [
        "#sparse categorical loss since we have scaler labels\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')\n",
        "\n",
        "#loss function helper\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "\n",
        "  return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n",
        "\n",
        "#accuracy function helper\n",
        "def accuracy_function(real, pred):\n",
        "  accuracies = tf.equal(real, tf.argmax(pred, axis=2))\n",
        "\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  accuracies = tf.math.logical_and(mask, accuracies)\n",
        "\n",
        "  accuracies = tf.cast(accuracies, dtype=tf.float32)\n",
        "  mask = tf.cast(mask, dtype=tf.float32)\n",
        "  return tf.reduce_sum(accuracies)/tf.reduce_sum(mask)\n",
        "\n",
        "#instantiate training loss and accuracy metrics\n",
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.Mean(name='train_accuracy')"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mp1oMp4Z3o74"
      },
      "source": [
        "### Training and Checkpointing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X43grOff3tr_"
      },
      "source": [
        "Instantiate transformer:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IhGyNAi23ru5"
      },
      "source": [
        "transformer = Transformer(\n",
        "    num_layers=num_layers,\n",
        "    d_model=d_model,\n",
        "    num_heads=num_heads,\n",
        "    dff=dff,\n",
        "    input_vocab_size=tokenizers.pt.get_vocab_size().numpy(),\n",
        "    target_vocab_size=tokenizers.en.get_vocab_size().numpy(),\n",
        "    pe_input=1000,\n",
        "    pe_target=1000,\n",
        "    rate=dropout_rate)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w_5YgNXo3xRR"
      },
      "source": [
        "Create checkpoint path and manager to save model after every n epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCpj2-Bi3wnz"
      },
      "source": [
        "checkpoint_path = \"./checkpoints/train\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "# if a checkpoint exists, restore the latest checkpoint.\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print('Latest checkpoint restored!!')"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JnNkrSO54c80"
      },
      "source": [
        "The target is divided into tar_inp (input) and tar_real (output). tar_inp is passed as an input to the decoder. `tar_real` is that same input shifted by 1: At each location in `tar_input`, `tar_real` contains the  next token that should be predicted.\n",
        "\n",
        "Since the transformer is an auto-regressive model, it makes perdictions one at at time, then uses its output so far to decide what to do next.\n",
        "\n",
        "During training we use [teacher-forcing](https://arxiv.org/abs/1610.09038) (ie pass true output to next time step regardless of what the model predicts at the current time step).\n",
        "\n",
        "When the transformer predicts each token, self-attention allows it to look at the previous tokens in the input sequence to make a better prediction for the next token.\n",
        "\n",
        "Note that we to prevent the model from looking at the expected output the model uses a look-ahead mask."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DJM9aAXC6BNr"
      },
      "source": [
        "EPOCHS = 20"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "El4EMG_G6Mjx"
      },
      "source": [
        "Training step"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHFcQ9FX6DU4"
      },
      "source": [
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "\n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer([inp, tar_inp],\n",
        "                                 training = True)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)\n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "\n",
        "  train_loss(loss)\n",
        "  train_accuracy(accuracy_function(tar_real, predictions))"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkxwnomT6Qkr"
      },
      "source": [
        "Training loop:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4dmGjOCt6Px_",
        "outputId": "ebe38e9c-90f4-4692-e137-641874f790a5"
      },
      "source": [
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "\n",
        "  train_loss.reset_states()\n",
        "  train_accuracy.reset_states()\n",
        "\n",
        "  # inp -> portuguese, tar -> english\n",
        "  for (batch, (inp, tar)) in enumerate(train_batches):\n",
        "    train_step(inp, tar)\n",
        "\n",
        "    if batch % 50 == 0:\n",
        "      print(f'Epoch {epoch + 1} Batch {batch} Loss {train_loss.result():.4f} Accuracy {train_accuracy.result():.4f}')\n",
        "\n",
        "  if (epoch + 1) % 5 == 0:\n",
        "    ckpt_save_path = ckpt_manager.save()\n",
        "    print(f'Saving checkpoint for epoch {epoch+1} at {ckpt_save_path}')\n",
        "\n",
        "  print(f'Epoch {epoch + 1} Loss {train_loss.result():.4f} Accuracy {train_accuracy.result():.4f}')\n",
        "\n",
        "  print(f'Time taken for epoch {epoch+1}: {time.time() - start:.2f} secs\\n')"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1 Batch 0 Loss 8.8879 Accuracy 0.0000\n",
            "Epoch 1 Batch 50 Loss 8.8201 Accuracy 0.0024\n",
            "Epoch 1 Batch 100 Loss 8.7131 Accuracy 0.0235\n",
            "Epoch 1 Batch 150 Loss 8.5960 Accuracy 0.0316\n",
            "Epoch 1 Batch 200 Loss 8.4548 Accuracy 0.0356\n",
            "Epoch 1 Batch 250 Loss 8.2849 Accuracy 0.0385\n",
            "Epoch 1 Batch 300 Loss 8.0944 Accuracy 0.0454\n",
            "Epoch 1 Batch 350 Loss 7.8951 Accuracy 0.0519\n",
            "Epoch 1 Batch 400 Loss 7.7070 Accuracy 0.0577\n",
            "Epoch 1 Batch 450 Loss 7.5374 Accuracy 0.0651\n",
            "Epoch 1 Batch 500 Loss 7.3897 Accuracy 0.0723\n",
            "Epoch 1 Batch 550 Loss 7.2550 Accuracy 0.0793\n",
            "Epoch 1 Batch 600 Loss 7.1303 Accuracy 0.0870\n",
            "Epoch 1 Batch 650 Loss 7.0154 Accuracy 0.0943\n",
            "Epoch 1 Batch 700 Loss 6.9066 Accuracy 0.1015\n",
            "Epoch 1 Batch 750 Loss 6.8074 Accuracy 0.1079\n",
            "Epoch 1 Batch 800 Loss 6.7151 Accuracy 0.1139\n",
            "Epoch 1 Loss 6.6988 Accuracy 0.1151\n",
            "Time taken for epoch 1: 231.90 secs\n",
            "\n",
            "Epoch 2 Batch 0 Loss 5.2716 Accuracy 0.1982\n",
            "Epoch 2 Batch 50 Loss 5.2709 Accuracy 0.2096\n",
            "Epoch 2 Batch 100 Loss 5.2266 Accuracy 0.2144\n",
            "Epoch 2 Batch 150 Loss 5.1779 Accuracy 0.2198\n",
            "Epoch 2 Batch 200 Loss 5.1442 Accuracy 0.2237\n",
            "Epoch 2 Batch 250 Loss 5.1162 Accuracy 0.2269\n",
            "Epoch 2 Batch 300 Loss 5.0922 Accuracy 0.2292\n",
            "Epoch 2 Batch 350 Loss 5.0758 Accuracy 0.2309\n",
            "Epoch 2 Batch 400 Loss 5.0522 Accuracy 0.2330\n",
            "Epoch 2 Batch 450 Loss 5.0292 Accuracy 0.2350\n",
            "Epoch 2 Batch 500 Loss 5.0085 Accuracy 0.2369\n",
            "Epoch 2 Batch 550 Loss 4.9876 Accuracy 0.2388\n",
            "Epoch 2 Batch 600 Loss 4.9680 Accuracy 0.2407\n",
            "Epoch 2 Batch 650 Loss 4.9503 Accuracy 0.2420\n",
            "Epoch 2 Batch 700 Loss 4.9326 Accuracy 0.2435\n",
            "Epoch 2 Batch 750 Loss 4.9138 Accuracy 0.2451\n",
            "Epoch 2 Batch 800 Loss 4.8976 Accuracy 0.2463\n",
            "Epoch 2 Loss 4.8934 Accuracy 0.2468\n",
            "Time taken for epoch 2: 211.73 secs\n",
            "\n",
            "Epoch 3 Batch 0 Loss 4.5978 Accuracy 0.2739\n",
            "Epoch 3 Batch 50 Loss 4.6094 Accuracy 0.2676\n",
            "Epoch 3 Batch 100 Loss 4.5961 Accuracy 0.2698\n",
            "Epoch 3 Batch 150 Loss 4.5787 Accuracy 0.2718\n",
            "Epoch 3 Batch 200 Loss 4.5706 Accuracy 0.2728\n",
            "Epoch 3 Batch 250 Loss 4.5597 Accuracy 0.2742\n",
            "Epoch 3 Batch 300 Loss 4.5431 Accuracy 0.2758\n",
            "Epoch 3 Batch 350 Loss 4.5328 Accuracy 0.2768\n",
            "Epoch 3 Batch 400 Loss 4.5210 Accuracy 0.2778\n",
            "Epoch 3 Batch 450 Loss 4.5103 Accuracy 0.2787\n",
            "Epoch 3 Batch 500 Loss 4.4952 Accuracy 0.2804\n",
            "Epoch 3 Batch 550 Loss 4.4823 Accuracy 0.2819\n",
            "Epoch 3 Batch 600 Loss 4.4686 Accuracy 0.2833\n",
            "Epoch 3 Batch 650 Loss 4.4547 Accuracy 0.2849\n",
            "Epoch 3 Batch 700 Loss 4.4436 Accuracy 0.2861\n",
            "Epoch 3 Batch 750 Loss 4.4286 Accuracy 0.2880\n",
            "Epoch 3 Batch 800 Loss 4.4150 Accuracy 0.2894\n",
            "Epoch 3 Loss 4.4118 Accuracy 0.2897\n",
            "Time taken for epoch 3: 213.34 secs\n",
            "\n",
            "Epoch 4 Batch 0 Loss 4.4374 Accuracy 0.2717\n",
            "Epoch 4 Batch 50 Loss 4.0882 Accuracy 0.3265\n",
            "Epoch 4 Batch 100 Loss 4.0646 Accuracy 0.3279\n",
            "Epoch 4 Batch 150 Loss 4.0613 Accuracy 0.3283\n",
            "Epoch 4 Batch 200 Loss 4.0476 Accuracy 0.3305\n",
            "Epoch 4 Batch 250 Loss 4.0369 Accuracy 0.3316\n",
            "Epoch 4 Batch 300 Loss 4.0204 Accuracy 0.3340\n",
            "Epoch 4 Batch 350 Loss 4.0044 Accuracy 0.3362\n",
            "Epoch 4 Batch 400 Loss 3.9902 Accuracy 0.3381\n",
            "Epoch 4 Batch 450 Loss 3.9768 Accuracy 0.3397\n",
            "Epoch 4 Batch 500 Loss 3.9607 Accuracy 0.3418\n",
            "Epoch 4 Batch 550 Loss 3.9448 Accuracy 0.3441\n",
            "Epoch 4 Batch 600 Loss 3.9300 Accuracy 0.3462\n",
            "Epoch 4 Batch 650 Loss 3.9160 Accuracy 0.3480\n",
            "Epoch 4 Batch 700 Loss 3.9009 Accuracy 0.3500\n",
            "Epoch 4 Batch 750 Loss 3.8867 Accuracy 0.3519\n",
            "Epoch 4 Batch 800 Loss 3.8733 Accuracy 0.3538\n",
            "Epoch 4 Loss 3.8713 Accuracy 0.3540\n",
            "Time taken for epoch 4: 212.34 secs\n",
            "\n",
            "Epoch 5 Batch 0 Loss 3.3881 Accuracy 0.4176\n",
            "Epoch 5 Batch 50 Loss 3.5499 Accuracy 0.3925\n",
            "Epoch 5 Batch 100 Loss 3.5363 Accuracy 0.3946\n",
            "Epoch 5 Batch 150 Loss 3.5173 Accuracy 0.3977\n",
            "Epoch 5 Batch 200 Loss 3.5133 Accuracy 0.3983\n",
            "Epoch 5 Batch 250 Loss 3.5028 Accuracy 0.3995\n",
            "Epoch 5 Batch 300 Loss 3.4983 Accuracy 0.4004\n",
            "Epoch 5 Batch 350 Loss 3.4911 Accuracy 0.4012\n",
            "Epoch 5 Batch 400 Loss 3.4785 Accuracy 0.4030\n",
            "Epoch 5 Batch 450 Loss 3.4655 Accuracy 0.4046\n",
            "Epoch 5 Batch 500 Loss 3.4583 Accuracy 0.4055\n",
            "Epoch 5 Batch 550 Loss 3.4516 Accuracy 0.4065\n",
            "Epoch 5 Batch 600 Loss 3.4435 Accuracy 0.4074\n",
            "Epoch 5 Batch 650 Loss 3.4330 Accuracy 0.4088\n",
            "Epoch 5 Batch 700 Loss 3.4231 Accuracy 0.4102\n",
            "Epoch 5 Batch 750 Loss 3.4118 Accuracy 0.4118\n",
            "Epoch 5 Batch 800 Loss 3.4045 Accuracy 0.4127\n",
            "Saving checkpoint for epoch 5 at ./checkpoints/train/ckpt-1\n",
            "Epoch 5 Loss 3.4022 Accuracy 0.4131\n",
            "Time taken for epoch 5: 213.17 secs\n",
            "\n",
            "Epoch 6 Batch 0 Loss 3.0727 Accuracy 0.4577\n",
            "Epoch 6 Batch 50 Loss 3.1260 Accuracy 0.4435\n",
            "Epoch 6 Batch 100 Loss 3.1199 Accuracy 0.4456\n",
            "Epoch 6 Batch 150 Loss 3.1158 Accuracy 0.4469\n",
            "Epoch 6 Batch 200 Loss 3.1086 Accuracy 0.4474\n",
            "Epoch 6 Batch 250 Loss 3.0977 Accuracy 0.4490\n",
            "Epoch 6 Batch 300 Loss 3.0945 Accuracy 0.4494\n",
            "Epoch 6 Batch 350 Loss 3.0802 Accuracy 0.4518\n",
            "Epoch 6 Batch 400 Loss 3.0674 Accuracy 0.4534\n",
            "Epoch 6 Batch 450 Loss 3.0582 Accuracy 0.4548\n",
            "Epoch 6 Batch 500 Loss 3.0461 Accuracy 0.4565\n",
            "Epoch 6 Batch 550 Loss 3.0383 Accuracy 0.4575\n",
            "Epoch 6 Batch 600 Loss 3.0267 Accuracy 0.4592\n",
            "Epoch 6 Batch 650 Loss 3.0198 Accuracy 0.4600\n",
            "Epoch 6 Batch 700 Loss 3.0119 Accuracy 0.4612\n",
            "Epoch 6 Batch 750 Loss 3.0057 Accuracy 0.4622\n",
            "Epoch 6 Batch 800 Loss 2.9979 Accuracy 0.4634\n",
            "Epoch 6 Loss 2.9968 Accuracy 0.4636\n",
            "Time taken for epoch 6: 211.26 secs\n",
            "\n",
            "Epoch 7 Batch 0 Loss 3.0440 Accuracy 0.4651\n",
            "Epoch 7 Batch 50 Loss 2.7326 Accuracy 0.4943\n",
            "Epoch 7 Batch 100 Loss 2.7475 Accuracy 0.4922\n",
            "Epoch 7 Batch 150 Loss 2.7337 Accuracy 0.4950\n",
            "Epoch 7 Batch 200 Loss 2.7282 Accuracy 0.4959\n",
            "Epoch 7 Batch 250 Loss 2.7160 Accuracy 0.4976\n",
            "Epoch 7 Batch 300 Loss 2.7124 Accuracy 0.4982\n",
            "Epoch 7 Batch 350 Loss 2.7045 Accuracy 0.4996\n",
            "Epoch 7 Batch 400 Loss 2.7006 Accuracy 0.5000\n",
            "Epoch 7 Batch 450 Loss 2.6940 Accuracy 0.5011\n",
            "Epoch 7 Batch 500 Loss 2.6851 Accuracy 0.5023\n",
            "Epoch 7 Batch 550 Loss 2.6791 Accuracy 0.5034\n",
            "Epoch 7 Batch 600 Loss 2.6739 Accuracy 0.5041\n",
            "Epoch 7 Batch 650 Loss 2.6711 Accuracy 0.5046\n",
            "Epoch 7 Batch 700 Loss 2.6672 Accuracy 0.5053\n",
            "Epoch 7 Batch 750 Loss 2.6633 Accuracy 0.5058\n",
            "Epoch 7 Batch 800 Loss 2.6596 Accuracy 0.5064\n",
            "Epoch 7 Loss 2.6584 Accuracy 0.5067\n",
            "Time taken for epoch 7: 212.79 secs\n",
            "\n",
            "Epoch 8 Batch 0 Loss 2.3865 Accuracy 0.5387\n",
            "Epoch 8 Batch 50 Loss 2.4189 Accuracy 0.5370\n",
            "Epoch 8 Batch 100 Loss 2.4395 Accuracy 0.5344\n",
            "Epoch 8 Batch 150 Loss 2.4465 Accuracy 0.5335\n",
            "Epoch 8 Batch 200 Loss 2.4423 Accuracy 0.5344\n",
            "Epoch 8 Batch 250 Loss 2.4388 Accuracy 0.5352\n",
            "Epoch 8 Batch 300 Loss 2.4355 Accuracy 0.5357\n",
            "Epoch 8 Batch 350 Loss 2.4364 Accuracy 0.5356\n",
            "Epoch 8 Batch 400 Loss 2.4345 Accuracy 0.5358\n",
            "Epoch 8 Batch 450 Loss 2.4301 Accuracy 0.5364\n",
            "Epoch 8 Batch 500 Loss 2.4260 Accuracy 0.5371\n",
            "Epoch 8 Batch 550 Loss 2.4235 Accuracy 0.5374\n",
            "Epoch 8 Batch 600 Loss 2.4224 Accuracy 0.5377\n",
            "Epoch 8 Batch 650 Loss 2.4213 Accuracy 0.5378\n",
            "Epoch 8 Batch 700 Loss 2.4181 Accuracy 0.5384\n",
            "Epoch 8 Batch 750 Loss 2.4172 Accuracy 0.5386\n",
            "Epoch 8 Batch 800 Loss 2.4158 Accuracy 0.5389\n",
            "Epoch 8 Loss 2.4159 Accuracy 0.5389\n",
            "Time taken for epoch 8: 212.00 secs\n",
            "\n",
            "Epoch 9 Batch 0 Loss 2.1817 Accuracy 0.5669\n",
            "Epoch 9 Batch 50 Loss 2.2261 Accuracy 0.5642\n",
            "Epoch 9 Batch 100 Loss 2.2232 Accuracy 0.5641\n",
            "Epoch 9 Batch 150 Loss 2.2311 Accuracy 0.5630\n",
            "Epoch 9 Batch 200 Loss 2.2400 Accuracy 0.5617\n",
            "Epoch 9 Batch 250 Loss 2.2420 Accuracy 0.5616\n",
            "Epoch 9 Batch 300 Loss 2.2422 Accuracy 0.5615\n",
            "Epoch 9 Batch 350 Loss 2.2432 Accuracy 0.5611\n",
            "Epoch 9 Batch 400 Loss 2.2439 Accuracy 0.5613\n",
            "Epoch 9 Batch 450 Loss 2.2408 Accuracy 0.5617\n",
            "Epoch 9 Batch 500 Loss 2.2399 Accuracy 0.5620\n",
            "Epoch 9 Batch 550 Loss 2.2386 Accuracy 0.5623\n",
            "Epoch 9 Batch 600 Loss 2.2372 Accuracy 0.5625\n",
            "Epoch 9 Batch 650 Loss 2.2359 Accuracy 0.5630\n",
            "Epoch 9 Batch 700 Loss 2.2365 Accuracy 0.5631\n",
            "Epoch 9 Batch 750 Loss 2.2344 Accuracy 0.5635\n",
            "Epoch 9 Batch 800 Loss 2.2344 Accuracy 0.5638\n",
            "Epoch 9 Loss 2.2348 Accuracy 0.5638\n",
            "Time taken for epoch 9: 211.88 secs\n",
            "\n",
            "Epoch 10 Batch 0 Loss 2.1256 Accuracy 0.5707\n",
            "Epoch 10 Batch 50 Loss 2.0756 Accuracy 0.5847\n",
            "Epoch 10 Batch 100 Loss 2.0967 Accuracy 0.5823\n",
            "Epoch 10 Batch 150 Loss 2.0985 Accuracy 0.5813\n",
            "Epoch 10 Batch 200 Loss 2.0944 Accuracy 0.5822\n",
            "Epoch 10 Batch 250 Loss 2.0946 Accuracy 0.5823\n",
            "Epoch 10 Batch 300 Loss 2.0917 Accuracy 0.5827\n",
            "Epoch 10 Batch 350 Loss 2.0888 Accuracy 0.5833\n",
            "Epoch 10 Batch 400 Loss 2.0893 Accuracy 0.5833\n",
            "Epoch 10 Batch 450 Loss 2.0865 Accuracy 0.5837\n",
            "Epoch 10 Batch 500 Loss 2.0891 Accuracy 0.5831\n",
            "Epoch 10 Batch 550 Loss 2.0890 Accuracy 0.5832\n",
            "Epoch 10 Batch 600 Loss 2.0900 Accuracy 0.5832\n",
            "Epoch 10 Batch 650 Loss 2.0925 Accuracy 0.5830\n",
            "Epoch 10 Batch 700 Loss 2.0933 Accuracy 0.5830\n",
            "Epoch 10 Batch 750 Loss 2.0934 Accuracy 0.5830\n",
            "Epoch 10 Batch 800 Loss 2.0949 Accuracy 0.5829\n",
            "Saving checkpoint for epoch 10 at ./checkpoints/train/ckpt-2\n",
            "Epoch 10 Loss 2.0942 Accuracy 0.5831\n",
            "Time taken for epoch 10: 211.58 secs\n",
            "\n",
            "Epoch 11 Batch 0 Loss 2.0828 Accuracy 0.5882\n",
            "Epoch 11 Batch 50 Loss 1.9557 Accuracy 0.6003\n",
            "Epoch 11 Batch 100 Loss 1.9641 Accuracy 0.5999\n",
            "Epoch 11 Batch 150 Loss 1.9542 Accuracy 0.6017\n",
            "Epoch 11 Batch 200 Loss 1.9651 Accuracy 0.6004\n",
            "Epoch 11 Batch 250 Loss 1.9702 Accuracy 0.5997\n",
            "Epoch 11 Batch 300 Loss 1.9687 Accuracy 0.6001\n",
            "Epoch 11 Batch 350 Loss 1.9702 Accuracy 0.6000\n",
            "Epoch 11 Batch 400 Loss 1.9699 Accuracy 0.6000\n",
            "Epoch 11 Batch 450 Loss 1.9719 Accuracy 0.5998\n",
            "Epoch 11 Batch 500 Loss 1.9709 Accuracy 0.6001\n",
            "Epoch 11 Batch 550 Loss 1.9748 Accuracy 0.5998\n",
            "Epoch 11 Batch 600 Loss 1.9758 Accuracy 0.5997\n",
            "Epoch 11 Batch 650 Loss 1.9774 Accuracy 0.5996\n",
            "Epoch 11 Batch 700 Loss 1.9800 Accuracy 0.5993\n",
            "Epoch 11 Batch 750 Loss 1.9816 Accuracy 0.5990\n",
            "Epoch 11 Batch 800 Loss 1.9818 Accuracy 0.5992\n",
            "Epoch 11 Loss 1.9822 Accuracy 0.5991\n",
            "Time taken for epoch 11: 214.58 secs\n",
            "\n",
            "Epoch 12 Batch 0 Loss 1.6608 Accuracy 0.6520\n",
            "Epoch 12 Batch 50 Loss 1.8485 Accuracy 0.6186\n",
            "Epoch 12 Batch 100 Loss 1.8588 Accuracy 0.6170\n",
            "Epoch 12 Batch 150 Loss 1.8643 Accuracy 0.6164\n",
            "Epoch 12 Batch 200 Loss 1.8641 Accuracy 0.6167\n",
            "Epoch 12 Batch 250 Loss 1.8672 Accuracy 0.6158\n",
            "Epoch 12 Batch 300 Loss 1.8697 Accuracy 0.6153\n",
            "Epoch 12 Batch 350 Loss 1.8737 Accuracy 0.6147\n",
            "Epoch 12 Batch 400 Loss 1.8734 Accuracy 0.6149\n",
            "Epoch 12 Batch 450 Loss 1.8754 Accuracy 0.6146\n",
            "Epoch 12 Batch 500 Loss 1.8762 Accuracy 0.6144\n",
            "Epoch 12 Batch 550 Loss 1.8762 Accuracy 0.6143\n",
            "Epoch 12 Batch 600 Loss 1.8789 Accuracy 0.6140\n",
            "Epoch 12 Batch 650 Loss 1.8819 Accuracy 0.6136\n",
            "Epoch 12 Batch 700 Loss 1.8811 Accuracy 0.6138\n",
            "Epoch 12 Batch 750 Loss 1.8828 Accuracy 0.6136\n",
            "Epoch 12 Batch 800 Loss 1.8855 Accuracy 0.6133\n",
            "Epoch 12 Loss 1.8864 Accuracy 0.6131\n",
            "Time taken for epoch 12: 210.74 secs\n",
            "\n",
            "Epoch 13 Batch 0 Loss 1.9717 Accuracy 0.6004\n",
            "Epoch 13 Batch 50 Loss 1.7784 Accuracy 0.6278\n",
            "Epoch 13 Batch 100 Loss 1.7884 Accuracy 0.6266\n",
            "Epoch 13 Batch 150 Loss 1.7843 Accuracy 0.6277\n",
            "Epoch 13 Batch 200 Loss 1.7910 Accuracy 0.6262\n",
            "Epoch 13 Batch 250 Loss 1.7876 Accuracy 0.6261\n",
            "Epoch 13 Batch 300 Loss 1.7892 Accuracy 0.6262\n",
            "Epoch 13 Batch 350 Loss 1.7925 Accuracy 0.6256\n",
            "Epoch 13 Batch 400 Loss 1.7927 Accuracy 0.6257\n",
            "Epoch 13 Batch 450 Loss 1.7949 Accuracy 0.6256\n",
            "Epoch 13 Batch 500 Loss 1.7967 Accuracy 0.6255\n",
            "Epoch 13 Batch 550 Loss 1.7991 Accuracy 0.6252\n",
            "Epoch 13 Batch 600 Loss 1.7998 Accuracy 0.6253\n",
            "Epoch 13 Batch 650 Loss 1.8032 Accuracy 0.6249\n",
            "Epoch 13 Batch 700 Loss 1.8046 Accuracy 0.6247\n",
            "Epoch 13 Batch 750 Loss 1.8057 Accuracy 0.6246\n",
            "Epoch 13 Batch 800 Loss 1.8060 Accuracy 0.6247\n",
            "Epoch 13 Loss 1.8059 Accuracy 0.6248\n",
            "Time taken for epoch 13: 212.71 secs\n",
            "\n",
            "Epoch 14 Batch 0 Loss 1.6354 Accuracy 0.6497\n",
            "Epoch 14 Batch 50 Loss 1.6800 Accuracy 0.6406\n",
            "Epoch 14 Batch 100 Loss 1.6971 Accuracy 0.6390\n",
            "Epoch 14 Batch 150 Loss 1.7038 Accuracy 0.6386\n",
            "Epoch 14 Batch 200 Loss 1.7061 Accuracy 0.6391\n",
            "Epoch 14 Batch 250 Loss 1.7083 Accuracy 0.6388\n",
            "Epoch 14 Batch 300 Loss 1.7126 Accuracy 0.6382\n",
            "Epoch 14 Batch 350 Loss 1.7160 Accuracy 0.6377\n",
            "Epoch 14 Batch 400 Loss 1.7201 Accuracy 0.6372\n",
            "Epoch 14 Batch 450 Loss 1.7197 Accuracy 0.6375\n",
            "Epoch 14 Batch 500 Loss 1.7192 Accuracy 0.6378\n",
            "Epoch 14 Batch 550 Loss 1.7209 Accuracy 0.6375\n",
            "Epoch 14 Batch 600 Loss 1.7250 Accuracy 0.6367\n",
            "Epoch 14 Batch 650 Loss 1.7266 Accuracy 0.6366\n",
            "Epoch 14 Batch 700 Loss 1.7310 Accuracy 0.6359\n",
            "Epoch 14 Batch 750 Loss 1.7341 Accuracy 0.6355\n",
            "Epoch 14 Batch 800 Loss 1.7368 Accuracy 0.6351\n",
            "Epoch 14 Loss 1.7369 Accuracy 0.6352\n",
            "Time taken for epoch 14: 212.31 secs\n",
            "\n",
            "Epoch 15 Batch 0 Loss 1.5594 Accuracy 0.6594\n",
            "Epoch 15 Batch 50 Loss 1.6293 Accuracy 0.6505\n",
            "Epoch 15 Batch 100 Loss 1.6245 Accuracy 0.6524\n",
            "Epoch 15 Batch 150 Loss 1.6287 Accuracy 0.6514\n",
            "Epoch 15 Batch 200 Loss 1.6362 Accuracy 0.6503\n",
            "Epoch 15 Batch 250 Loss 1.6460 Accuracy 0.6489\n",
            "Epoch 15 Batch 300 Loss 1.6496 Accuracy 0.6482\n",
            "Epoch 15 Batch 350 Loss 1.6549 Accuracy 0.6475\n",
            "Epoch 15 Batch 400 Loss 1.6574 Accuracy 0.6467\n",
            "Epoch 15 Batch 450 Loss 1.6571 Accuracy 0.6468\n",
            "Epoch 15 Batch 500 Loss 1.6602 Accuracy 0.6463\n",
            "Epoch 15 Batch 550 Loss 1.6637 Accuracy 0.6458\n",
            "Epoch 15 Batch 600 Loss 1.6640 Accuracy 0.6458\n",
            "Epoch 15 Batch 650 Loss 1.6679 Accuracy 0.6453\n",
            "Epoch 15 Batch 700 Loss 1.6697 Accuracy 0.6451\n",
            "Epoch 15 Batch 750 Loss 1.6721 Accuracy 0.6446\n",
            "Epoch 15 Batch 800 Loss 1.6741 Accuracy 0.6444\n",
            "Saving checkpoint for epoch 15 at ./checkpoints/train/ckpt-3\n",
            "Epoch 15 Loss 1.6739 Accuracy 0.6445\n",
            "Time taken for epoch 15: 212.64 secs\n",
            "\n",
            "Epoch 16 Batch 0 Loss 1.4678 Accuracy 0.6711\n",
            "Epoch 16 Batch 50 Loss 1.5927 Accuracy 0.6569\n",
            "Epoch 16 Batch 100 Loss 1.5860 Accuracy 0.6579\n",
            "Epoch 16 Batch 150 Loss 1.5901 Accuracy 0.6573\n",
            "Epoch 16 Batch 200 Loss 1.5906 Accuracy 0.6568\n",
            "Epoch 16 Batch 250 Loss 1.5893 Accuracy 0.6570\n",
            "Epoch 16 Batch 300 Loss 1.5921 Accuracy 0.6569\n",
            "Epoch 16 Batch 350 Loss 1.5952 Accuracy 0.6565\n",
            "Epoch 16 Batch 400 Loss 1.6001 Accuracy 0.6558\n",
            "Epoch 16 Batch 450 Loss 1.6010 Accuracy 0.6555\n",
            "Epoch 16 Batch 500 Loss 1.6024 Accuracy 0.6552\n",
            "Epoch 16 Batch 550 Loss 1.6077 Accuracy 0.6544\n",
            "Epoch 16 Batch 600 Loss 1.6104 Accuracy 0.6542\n",
            "Epoch 16 Batch 650 Loss 1.6135 Accuracy 0.6537\n",
            "Epoch 16 Batch 700 Loss 1.6162 Accuracy 0.6533\n",
            "Epoch 16 Batch 750 Loss 1.6184 Accuracy 0.6530\n",
            "Epoch 16 Batch 800 Loss 1.6206 Accuracy 0.6527\n",
            "Epoch 16 Loss 1.6211 Accuracy 0.6527\n",
            "Time taken for epoch 16: 213.33 secs\n",
            "\n",
            "Epoch 17 Batch 0 Loss 1.5519 Accuracy 0.6558\n",
            "Epoch 17 Batch 50 Loss 1.5301 Accuracy 0.6655\n",
            "Epoch 17 Batch 100 Loss 1.5340 Accuracy 0.6654\n",
            "Epoch 17 Batch 150 Loss 1.5384 Accuracy 0.6652\n",
            "Epoch 17 Batch 200 Loss 1.5355 Accuracy 0.6660\n",
            "Epoch 17 Batch 250 Loss 1.5414 Accuracy 0.6654\n",
            "Epoch 17 Batch 300 Loss 1.5446 Accuracy 0.6649\n",
            "Epoch 17 Batch 350 Loss 1.5528 Accuracy 0.6635\n",
            "Epoch 17 Batch 400 Loss 1.5529 Accuracy 0.6634\n",
            "Epoch 17 Batch 450 Loss 1.5555 Accuracy 0.6629\n",
            "Epoch 17 Batch 500 Loss 1.5580 Accuracy 0.6625\n",
            "Epoch 17 Batch 550 Loss 1.5587 Accuracy 0.6625\n",
            "Epoch 17 Batch 600 Loss 1.5617 Accuracy 0.6620\n",
            "Epoch 17 Batch 650 Loss 1.5638 Accuracy 0.6616\n",
            "Epoch 17 Batch 700 Loss 1.5650 Accuracy 0.6616\n",
            "Epoch 17 Batch 750 Loss 1.5678 Accuracy 0.6611\n",
            "Epoch 17 Batch 800 Loss 1.5710 Accuracy 0.6606\n",
            "Epoch 17 Loss 1.5723 Accuracy 0.6604\n",
            "Time taken for epoch 17: 213.89 secs\n",
            "\n",
            "Epoch 18 Batch 0 Loss 1.3702 Accuracy 0.6790\n",
            "Epoch 18 Batch 50 Loss 1.4787 Accuracy 0.6739\n",
            "Epoch 18 Batch 100 Loss 1.4806 Accuracy 0.6736\n",
            "Epoch 18 Batch 150 Loss 1.4824 Accuracy 0.6738\n",
            "Epoch 18 Batch 200 Loss 1.4832 Accuracy 0.6742\n",
            "Epoch 18 Batch 250 Loss 1.4855 Accuracy 0.6739\n",
            "Epoch 18 Batch 300 Loss 1.4897 Accuracy 0.6731\n",
            "Epoch 18 Batch 350 Loss 1.4936 Accuracy 0.6724\n",
            "Epoch 18 Batch 400 Loss 1.4988 Accuracy 0.6713\n",
            "Epoch 18 Batch 450 Loss 1.5027 Accuracy 0.6708\n",
            "Epoch 18 Batch 500 Loss 1.5041 Accuracy 0.6708\n",
            "Epoch 18 Batch 550 Loss 1.5082 Accuracy 0.6702\n",
            "Epoch 18 Batch 600 Loss 1.5132 Accuracy 0.6692\n",
            "Epoch 18 Batch 650 Loss 1.5155 Accuracy 0.6688\n",
            "Epoch 18 Batch 700 Loss 1.5207 Accuracy 0.6680\n",
            "Epoch 18 Batch 750 Loss 1.5229 Accuracy 0.6676\n",
            "Epoch 18 Batch 800 Loss 1.5269 Accuracy 0.6671\n",
            "Epoch 18 Loss 1.5273 Accuracy 0.6670\n",
            "Time taken for epoch 18: 212.90 secs\n",
            "\n",
            "Epoch 19 Batch 0 Loss 1.4530 Accuracy 0.6796\n",
            "Epoch 19 Batch 50 Loss 1.4427 Accuracy 0.6810\n",
            "Epoch 19 Batch 100 Loss 1.4302 Accuracy 0.6836\n",
            "Epoch 19 Batch 150 Loss 1.4391 Accuracy 0.6816\n",
            "Epoch 19 Batch 200 Loss 1.4511 Accuracy 0.6794\n",
            "Epoch 19 Batch 250 Loss 1.4567 Accuracy 0.6783\n",
            "Epoch 19 Batch 300 Loss 1.4617 Accuracy 0.6776\n",
            "Epoch 19 Batch 350 Loss 1.4671 Accuracy 0.6768\n",
            "Epoch 19 Batch 400 Loss 1.4686 Accuracy 0.6765\n",
            "Epoch 19 Batch 450 Loss 1.4711 Accuracy 0.6762\n",
            "Epoch 19 Batch 500 Loss 1.4724 Accuracy 0.6759\n",
            "Epoch 19 Batch 550 Loss 1.4749 Accuracy 0.6755\n",
            "Epoch 19 Batch 600 Loss 1.4773 Accuracy 0.6752\n",
            "Epoch 19 Batch 650 Loss 1.4799 Accuracy 0.6748\n",
            "Epoch 19 Batch 700 Loss 1.4819 Accuracy 0.6745\n",
            "Epoch 19 Batch 750 Loss 1.4850 Accuracy 0.6740\n",
            "Epoch 19 Batch 800 Loss 1.4880 Accuracy 0.6735\n",
            "Epoch 19 Loss 1.4880 Accuracy 0.6736\n",
            "Time taken for epoch 19: 212.62 secs\n",
            "\n",
            "Epoch 20 Batch 0 Loss 1.2712 Accuracy 0.7145\n",
            "Epoch 20 Batch 50 Loss 1.3952 Accuracy 0.6894\n",
            "Epoch 20 Batch 100 Loss 1.4023 Accuracy 0.6881\n",
            "Epoch 20 Batch 150 Loss 1.4063 Accuracy 0.6871\n",
            "Epoch 20 Batch 200 Loss 1.4137 Accuracy 0.6855\n",
            "Epoch 20 Batch 250 Loss 1.4209 Accuracy 0.6842\n",
            "Epoch 20 Batch 300 Loss 1.4254 Accuracy 0.6830\n",
            "Epoch 20 Batch 350 Loss 1.4251 Accuracy 0.6830\n",
            "Epoch 20 Batch 400 Loss 1.4270 Accuracy 0.6828\n",
            "Epoch 20 Batch 450 Loss 1.4323 Accuracy 0.6819\n",
            "Epoch 20 Batch 500 Loss 1.4359 Accuracy 0.6815\n",
            "Epoch 20 Batch 550 Loss 1.4384 Accuracy 0.6813\n",
            "Epoch 20 Batch 600 Loss 1.4442 Accuracy 0.6802\n",
            "Epoch 20 Batch 650 Loss 1.4463 Accuracy 0.6799\n",
            "Epoch 20 Batch 700 Loss 1.4489 Accuracy 0.6793\n",
            "Epoch 20 Batch 750 Loss 1.4500 Accuracy 0.6793\n",
            "Epoch 20 Batch 800 Loss 1.4516 Accuracy 0.6791\n",
            "Saving checkpoint for epoch 20 at ./checkpoints/train/ckpt-4\n",
            "Epoch 20 Loss 1.4527 Accuracy 0.6790\n",
            "Time taken for epoch 20: 212.45 secs\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ty5NgVa469ON"
      },
      "source": [
        "### Inferences\n",
        "\n",
        "The following steps are used for inference:\n",
        "\n",
        "* Encode the input sentence using the Portuguese tokenizer (`tokenizers.pt`). This is the encoder input.\n",
        "* The decoder input is initialized to the `[START]` token.\n",
        "* Calculate the padding masks and the look ahead masks.\n",
        "* The `decoder` then outputs the predictions by looking at the `encoder output` and its own output (self-attention).\n",
        "* Concatenate the predicted token to the decoder input and pass it to the decoder.\n",
        "* In this approach, the decoder predicts the next token based on the previous tokens it predicted."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NGaHaWGI_rpe"
      },
      "source": [
        "class Translator(tf.Module):\n",
        "  def __init__(self, tokenizers, transformer):\n",
        "    self.tokenizers = tokenizers\n",
        "    self.transformer = transformer\n",
        "\n",
        "  def __call__(self, sentence, max_length=20):\n",
        "    # input sentence is portuguese, hence adding the start and end token\n",
        "    assert isinstance(sentence, tf.Tensor)\n",
        "    if len(sentence.shape) == 0:\n",
        "      sentence = sentence[tf.newaxis]\n",
        "\n",
        "    sentence = self.tokenizers.pt.tokenize(sentence).to_tensor()\n",
        "    \n",
        "    encoder_input = sentence\n",
        "\n",
        "    # as the target is english, the first token to the transformer should be the\n",
        "    # english start token.\n",
        "    start_end = self.tokenizers.en.tokenize([''])[0]\n",
        "    start = start_end[0][tf.newaxis]\n",
        "    end = start_end[1][tf.newaxis]\n",
        "\n",
        "    # `tf.TensorArray` is required here (instead of a python list) so that the\n",
        "    # dynamic-loop can be traced by `tf.function`.\n",
        "    output_array = tf.TensorArray(dtype=tf.int64, size=0, dynamic_size=True)\n",
        "    output_array = output_array.write(0, start)\n",
        "    \n",
        "    for i in tf.range(max_length):\n",
        "      output = tf.transpose(output_array.stack())\n",
        "      predictions, _ = self.transformer([encoder_input, output], training=False)\n",
        "      \n",
        "      # select the last token from the seq_len dimension\n",
        "      predictions = predictions[:, -1:, :]  # (batch_size, 1, vocab_size)\n",
        "\n",
        "      predicted_id = tf.argmax(predictions, axis=-1)\n",
        "\n",
        "      # concatentate the predicted_id to the output which is given to the decoder\n",
        "      # as its input.\n",
        "      output_array = output_array.write(i+1, predicted_id[0])\n",
        "\n",
        "      if predicted_id == end:\n",
        "        break\n",
        "\n",
        "    output = tf.transpose(output_array.stack())\n",
        "    # output.shape (1, tokens)\n",
        "    text = tokenizers.en.detokenize(output)[0]  # shape: ()\n",
        "\n",
        "    tokens = tokenizers.en.lookup(output)[0]\n",
        "\n",
        "    # `tf.function` prevents us from using the attention_weights that were\n",
        "    # calculated on the last iteration of the loop. So recalculate them outside\n",
        "    # the loop.\n",
        "    _, attention_weights = self.transformer([encoder_input, output[:,:-1]], training=False)\n",
        "\n",
        "    return text, tokens, attention_weights"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WZrKtloxAUo-"
      },
      "source": [
        "Instantiate translater and test:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UwPxPVlzAnAu"
      },
      "source": [
        "translator = Translator(tokenizers, transformer)\n",
        "\n",
        "#helper function for testing examples\n",
        "def print_translation(sentence, tokens, ground_truth):\n",
        "  print(f'{\"Input:\":15s}: {sentence}')\n",
        "  print(f'{\"Prediction\":15s}: {tokens.numpy().decode(\"utf-8\")}')\n",
        "  print(f'{\"Ground truth\":15s}: {ground_truth}')"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N2PYykzLArZA",
        "outputId": "9ce6e61e-712b-4dcb-ebe5-7e581ffe2003"
      },
      "source": [
        "sentence = \"este é um problema que temos que resolver.\"\n",
        "ground_truth = \"this is a problem we have to solve .\"\n",
        "\n",
        "translated_text, translated_tokens, attention_weights = translator(\n",
        "    tf.constant(sentence))\n",
        "print_translation(sentence, translated_text, ground_truth)\n",
        "print()\n",
        "\n",
        "sentence = \"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\"\n",
        "ground_truth = \"so i \\'ll just share with you some stories very quickly of some magical things that have happened .\"\n",
        "\n",
        "translated_text, translated_tokens, attention_weights = translator(\n",
        "    tf.constant(sentence))\n",
        "print_translation(sentence, translated_text, ground_truth)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input:         : este é um problema que temos que resolver.\n",
            "Prediction     : this is a problem that we have to solve .\n",
            "Ground truth   : this is a problem we have to solve .\n",
            "\n",
            "Input:         : vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\n",
            "Prediction     : so i ' m going to share a lot of quick share with you some stories of some magic things\n",
            "Ground truth   : so i 'll just share with you some stories very quickly of some magical things that have happened .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AIBkyIgUA_DU"
      },
      "source": [
        "### Attention Plots\n",
        "\n",
        "Since the Translator class returns a dictionary of attention maps, we can visualize the internal workings of the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MkEH8Tx1BL8g"
      },
      "source": [
        "Example case:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Da0uEi8nBJ_n",
        "outputId": "0bc73fe1-499a-46e9-8922-589834b1922a"
      },
      "source": [
        "sentence = \"este é o primeiro livro que eu fiz.\"\n",
        "ground_truth = \"this is the first book i've ever done.\"\n",
        "\n",
        "translated_text, translated_tokens, attention_weights = translator(\n",
        "    tf.constant(sentence))\n",
        "print_translation(sentence, translated_text, ground_truth)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input:         : este é o primeiro livro que eu fiz.\n",
            "Prediction     : this is the first book i did .\n",
            "Ground truth   : this is the first book i've ever done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CUtv4c1vBQXT"
      },
      "source": [
        "Helper function to plot attention plot:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8t7kD3cnBT7w"
      },
      "source": [
        "def plot_attention_head(in_tokens, translated_tokens, attention):\n",
        "  # The plot is of the attention when a token was generated.\n",
        "  # The model didn't generate `<START>` in the output. Skip it.\n",
        "  translated_tokens = translated_tokens[1:]\n",
        "\n",
        "  ax = plt.gca()\n",
        "  ax.matshow(attention)\n",
        "  ax.set_xticks(range(len(in_tokens)))\n",
        "  ax.set_yticks(range(len(translated_tokens)))\n",
        "\n",
        "  labels = [label.decode('utf-8') for label in in_tokens.numpy()]\n",
        "  ax.set_xticklabels(\n",
        "      labels, rotation=90)\n",
        "\n",
        "  labels = [label.decode('utf-8') for label in translated_tokens.numpy()]\n",
        "  ax.set_yticklabels(labels)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BvBfLA2rBvlL"
      },
      "source": [
        "Attention plot for head:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "id": "LjPsdGY1B0fi",
        "outputId": "9266c8ba-554a-4804-ea7c-ffedc8a3d862"
      },
      "source": [
        "#attention\n",
        "head = 0\n",
        "# shape: (batch=1, num_heads, seq_len_q, seq_len_k)\n",
        "attention_heads = tf.squeeze(\n",
        "  attention_weights['decoder_layer4_block2'], 0)\n",
        "attention = attention_heads[head]\n",
        "\n",
        "#in_tokens\n",
        "in_tokens = tf.convert_to_tensor([sentence])\n",
        "in_tokens = tokenizers.pt.tokenize(in_tokens).to_tensor()\n",
        "in_tokens = tokenizers.pt.lookup(in_tokens)[0]\n",
        "\n",
        "#plot\n",
        "plot_attention_head(in_tokens, translated_tokens, attention)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEcCAYAAABaqQgwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXnElEQVR4nO3deZhlVX3u8e9LN0NjMyigiAItGBkkCk2TCAK2eBNRSRzRgN5cTWIHkYAmShqnXH3EhxuvkgskYkuYLuoV9TqhiIrB1saATdPQKDbGqHEChIDMPRRv/ti7mkNRp2s4e59Tddb7eZ566px9Tv3W6qqut/a01pJtIiJKsMWgOxAR0S8JvIgoRgIvIoqRwIuIYiTwIqIYCbyIKEYCLyKKkcCLiGLMHXQHYjhJ2gp4Rv10re0Ng+xPBIAy0iKaJmkxcBHwU0DA7sD/sL18gN2KSOBF8yRdBxxve239/BnAJ20fPNieRelyDi/asOVo2AHYvgXYcoD9iQCG7ByepLMm8bZ7bL+r9c6U7TpJ5wGX1M9fC6wcYH8igCE7pJX0M+A9E7xtqe39+tGfUknaGngzcHi96dvAP9leN7heRQzZHh5wpu2LNvcGSY/vV2dKJGkOcIPtfYEPD7o/EZ2G7RzexoneYPsf+tGRUtkeAdZK2mPQfYkYa9gOaVfZXjjofpRO0nLgIOBa4P7R7bb/eGCdimD4DmljZnj3oDsQMZ5h28PbCDww3kuAbW/f5y5FxAwybHt4a2wfNOhOlErSd2wfLuleoPMvaf7gxIwwbIEXA2T78PrzdoPuS8R4hi3wPj3eRkl/CLzd9h/0uT/FknQ48Du2L5C0M7Cd7Z8Mul8x87U5gGDYAu9fJd0C7AZ8HvhfwAVUh1SnD7JjJZH0d8AiYB+q7/9WVKMunjvIfsWs8VImMYAAKD7wPgQsAb4LvKj+vNT2OQPtVXleTnVbyioA27+SlMPcmKzWBhAM21Xa6zsvWkhaa3ufQfapRJKutf17o/dFSnoc8F3bzxp036Jsw7aHt4OkV3Q8n9v53Pb/H0CfSnSppI8CO0p6I/BnwMcG3KeYRSQ9H/grqtMiADcD59i+qqe6Q7aHd8FmXrbtP+tbZwon6Q+AP6Q6f3qF7a8PuEsxS0h6CXAO8D6q0yICFlKdszvJ9lemXXuYAm9zJD3J9m2D7kdJJG1Px1GE7f8cYHdilpB0FXCK7RvGbH8WcLbt50239rAd0j6KpB2BVwLHA/tRXb2Nlkn6S+C9wEPAw9Q3HgN7DbJfMWvsOjbsAGzfKOlJvRQeusCTNI/qsvbxVFcKtwNeBjSynkL9Df8AsJvtF0naHzjU9j83UX9IvA04wPYdbRSX9FTgbKr59kw1394ptn/RRnvRd/dP87UJDdUhraRPAEcAXwP+H/BN4N9sP63BNi6nurfsnbafLWkucL3t322qjdlO0leBV9geb1xzE/W/DnwC+L/1ptcBr82N5cNB0t2Mv4Mi4HDb057Tctj28PYH7qK6onOz7RFJTSf6zrYvlXQagO2NkkYabmO2Ow24WtI1wKZZjm2f3FD9XWx3XqC6UNJbGqoNgKQ9qUaKfKM+aphr+94m24iuXrqZ1/53L4WHKvBsHyhpX+A44BuS7gC2a/iCxf2SdqIeHC/pOcBvG6o9LD5KtXe9huocXtPulPQ64JP18+OAO5sqXt9KswR4ArA38FTgXOAFTbUR3dn+Vlu1h+2Q9jm2/7Xj+cFUvwyvBn5h+7AG2lhIdf7oAOAmYBfg2PFOspZq7A3gLdTfk+pncCjVH56rgZNt/0dD9VcDvwdcM/rvkLQmpy36Q9K/8OjZdjrZ9rT/8Axb4I0747EkAUc0sRB0vUDNCNUNkQLWAltkgZpHSPoA1SLcX+LRh7Q935ZSr5lxse3X9lprM21cY/v3R4O7Pk+7KiNF+qPeURnrOcCpwO22D5lu7aE6pO3GVao3ter9d+tQ/f7oBkmrqG6MjMpx9efTOrY1cltKfV52T0lb2V7fa70uviXpHcC8+gbqE6nCO/rA9nWjjyU9j2oG7W2AE2xf3kvtYQu8vSR9sduLvaypIGlX4ClUvwQHUe3dAWwPbDvdusOoyaviXfw7sKL+WXeumdHUKmlLgT+nOgf5l8BXgPMaqh2TIOmFVCMr1gGn2/6XJuoOW+D9hmrGlDa8EHg91QnsD/FI4N0LvKOlNmcVSUfZ/uaY8cybNDiW+cf1xxZU91k2yvbDVGN/M/53ACR9j+rc+AepZjwaPXcOgO1V0649ZOfwWj1ZXrfxStufbbON2UrSe23/XZcxzY2NZZa0sJf/9JOo/xPGOWluu5GRIvX3Z7z6TX1/jgCurpfMHN3W6vesSfXQss1dtDhqurWHbQ+vHzPqPrUeI3ov1R7AQqo5977Wh7ZntDrstgAut31pi019qD7F8BngU7Zvarj+oo7H2wDHUt2i0pTLxtR/OfCrButfAXxP0rG2b6+3nccsOc9se3GbxYfmAziEahze6PM/Bb4AnAU8oaE2bqg/vxD4HPBMqit4A//3z5QPYGUf2tgVOBlYQXWu7V0tt3ddi7W3oNoja6re9cAxVDfgHza6bdD/L6bQ/1M7Hh875rUP9FJ7i9aSdDA+CqwHkHQkcAZwMdWNwcsaamP03N1LqG6P+H7Htqh8Q9LbJO0u6QmjH002YPtW22cBJwCrmXhK8EmTtLDjY5GkE2j3aOh3gCc2WM+2LwP+GDhH0kl0P0Scif6k4/FpY147upfCw3ZIO8eP3Ov1GmCZq/Ntn61vJm3CdZKuoLrFYmk9dXkbowlms9dQ/YKdOGZ7U+fA9qvbeCXVCItPAX/TRO3ah3gkIDZS3VN4bFPFx1nG8laqe8waawLA9o/qP/znA7PpHkJ1eTze8ykZusCTNNf2RqphQEs6Xmvq3/rnVJfLf2D7AUl7AI2N46xvkn4tsJft99X1d7V9bVNt1O08m2qiBYBvu9mRIvtThV3nbCbnNlj/fKqQe6HtJs99jbqMqt+jv1wGjql+NI3c/rID1c/4aZ0/4x5rbuKOC3e27wNeXbcxW7jL4/GeT8mwXaV9J/Bi4A5gD2ChbUt6OnCR7Z5XzZL0Eao9uqNs71cvJvI193D3dz/r122cArwRGL1N5OVUe8NnN1T/UuAe4OP1puOBHWy/uon6batn3TmE6vyvgD8CrgV+BGD7vT3Wb+VnLOlU238v6WzGvwrc1OQNraon47if6ns/DxiddUfANra3nHbtYQo82DSY/8lU/4Hur7c9A5jvBi7L65GFaTbdAiPpBtvP7rV2P+rX9W6kmsNv9PvT6CI7kn5ge/+Jtk2j7qW2Xy1pDY/+hRbVeaum+r8ceInr2VHq0xZftn1kQ/Vb+RlLutP2TvXMMXeNfd0TrARWgqE6pO02ltb2LRO9Zwo21OM5R2dL2YVmz+G1XR+qgOic0mqEZi+8rOqcyEHS7wMrG6h7Sv35mAZqbc6TqC9+1dbX25rS1s/4Nkm7AW8AFjNLL6ZN5nd0ur/HQxV4wH713ks3ojp/0ouzqG5HeaKk04FXMY0FgQdYH6oJTK+R9Ln6+cuAJmdsPphqPrzR2Uv2ANaO7plNd0/M9q/rzz9rpptdXQxcO+b7c2GD9dv6GX8EuJLq4tB1Hdv7MsW+pF1t39pAqdZ+j4fqkFbVtEETGXGPU4GrmnPvBVTf+Ctt39xLvX7Xr9tYSHVRAaqLFtc3WHuzP4fpBtY4Vzc3vVSV9fbTqdulrYU8clFneZPfn7p+az9jSR+x/aam6k2h3S/bfkkDdVr7PR6qwIuI2Jxhu/E4IqKrBF5EFGPoA0/SkonflfqpP7g2Ur9/9Yc+8Hj0aIvUT/2Z2Ebq96l+CYEXEQHMwqu0W82Z53lzJ3/3wfqRB9lqzrxJv//J+9w9pf7cfefD7LjT5P9u/Pqm+VOqv551bMXWU/qaqfxMN7COLadYfypme/1+tJH6zdZ/iPtZ73Xj3nQ96248njd3ew7brbUFq3jHF7/QWm2A9+/f80qRE3r4oYdabyNiprrGV3Z9LYe0EVGMBF5EFCOBFxHFSOBFRDESeBFRjAReRBSjscCTtKOkE+vHiyVd1uV950nqaebbiIjpaHIPb0ceu0rVY9j+C9s/aLDdiIhJaTLwzgD2rpdD/CAwX9JnJP1Q0sfr1biQdFW91uccSRdKuknSGklvbbAvERGP0eRIi6XAAbYPlLSYasWnZwK/olod/rnAdzrefyDwFNsHQHVI3GBfIiIeo82LFtfa/oXth6lWhl8w5vV/B/aSdLako6mW9RuXpCWSVkpauX7kwfZ6HBFDrc3AW9fxeIQxe5O27wKeDVwFnACc162Q7WW2F9leNJWJACIiOjV5SHsvsN1k3yxpZ2C97c9KWgtc0mBfIiIeo7HAs32npBWSbgIeBG6b4EueAlwgaXQv87Sm+hIRMZ5Gp4eyfXyX7Sd1PF7c8VIvC2JHRExJRlpERDESeBFRjAReRBQjgRcRxUjgRUQxEngRUYxZt2pZ25b+zZtarX/bhe0PjXvacWtab6NNc5/0xFbrb7x1oltEY1hlDy8iipHAi4hiJPAiohgJvIgoRgIvIoqRwIuIYiTwIqIYCbyIKMZAAk/S1YNoNyLKNpDAs33YINqNiLINag/vvvrzkyUtl7S6Xp/2iEH0JyLKMOixtMcDV9g+XdIcYNsB9ycihtigA+97wPmStgQ+b3v1eG+StARYArDNnEkvjBYR8SgDvUprezlwJPBL4EJJf9rlfVmXNiJ6NtDAk7QncJvtj1EtxJ1VzCKiNYM+pF0MvF3SBuA+YNw9vIiIJgwk8GzPrz9fBFw0iD5ERHky0iIiipHAi4hiJPAiohgJvIgoRgIvIoqRwIuIYgz6PrxpEEitVX/cZde3Vhvg6cvnt1ofwPPbbeNH5z691fq7Xzyn1fpb33Fnq/U9MtJqfex26w+x7OFFRDESeBFRjAReRBQjgRcRxUjgRUQxEngRUYwEXkQUI4EXEcVoPPAk7SjpxPrxYkmXNd1GRMR0tLGHtyNwYgt1IyJ60sbQsjOAvSWtBjYA90v6DHAAcB3wOtuWdDDwYWA+cAfwetu/bqE/ERFAO3t4S4Ef2z4QeDtwEPAWYH9gL+C59bKMZwOvsn0wcD5wegt9iYjYpB+TB1xr+xcA9V7fAuBuqj2+r6uaCGAO0HXvLuvSRkQT+hF46zoej9RtCvi+7UMnU8D2MmAZwA5b75qpIiJiWto4pL0XmGg3bC2wi6RDASRtKemZLfQlImKTxvfwbN8paYWkm4AHgdvGec96Sa8CzpK0Q92PfwC+33R/IiJGtXJIa/v4LttP6ni8GjiyjfYjIsaTkRYRUYwEXkQUI4EXEcVI4EVEMRJ4EVGMBF5EFCOBFxHFmHULcXv9ejb+7OctNtDuyLWRu37ban0AHm53IeiLDv3nVut/4N2vaLX+xo0bW60fM1f28CKiGAm8iChGAi8iipHAi4hiJPAiohgJvIgoRgIvIoqRwIuIYjQWeJJOlnSzpLskLZ3C1y2QNO6EoRERTWpypMWJwH8bXaFsLElzbY93i/sC4HjgEw32JSLiMRoJPEnnUq05e7mk84G9bZ8k6ULgIaq1aVdI+gLwf+ovM9UU72cA+9VLOF5k+8wm+hQRMVYjgWf7BElHA88Hjhnz8lOBw2yPSPoS8GbbKyTNpwrDpcDbbI/9uk0etS4t2zbR5YgoUD8uWnza9uho9hXAhyWdDOzY5RD3MWwvs73I9qIt2bq1jkbEcOtH4N0/+sD2GcBfAPOoDnH37UP7ERFAn6eHkrS37TXAGkmHAPsCP2fihbsjInrW7/vw3iLpJkk3AhuAy4EbgRFJN0h6a5/7ExEFaWwPz/aC+uGF9Qe2Xz/mPX/V5cuPaqofERHdZKRFRBQjgRcRxUjgRUQxEngRUYwEXkQUI4EXEcWYdevSAq2vHduqlteM7Yf37X1wq/Wv+OXnW61/9B6LWq3vrHs7Y2UPLyKKkcCLiGIk8CKiGAm8iChGAi8iipHAi4hiJPAiohgJvIgoxrQCr15L9qZeG5f0U0k791onImIysocXEcXoJfDmSvq4pJslfUbStpJeIOl6SWsknS9pa4Bu20dJmifpcklv7OlfExGxGb0E3j7AP9neD7gH+Guqqd1fY/t3qcbpvknSNuNt76gzH/gS8EnbH+uhPxERm9VL4P3c9or68SXAC4Cf2L6l3nYRcCRVMI63fdQXgAtsX9ytIUlLJK2UtHID63rockSUrJfAGztlyd3TrLMCOFqSujaUhbgjogG9BN4ekg6tHx8PrAQWSHp6ve2/A98C1nbZPuo9wF3AP/bQl4iICfUSeGuBN0u6GXg8cCbwBuDTktYADwPn2n5ovO1jap0CzJP09z30JyJis6Y1AajtnwL7jvPSlcBB47y/2/YFHU/fMJ2+RERMVu7Di4hiJPAiohgJvIgoRgIvIoqRwIuIYiTwIqIYs3Nd2hhqba8bu/6oA1ut/9OXzmm1/p5fbH9t462/eWOr9bXVlu3VfqD7flz28CKiGAm8iChGAi8iipHAi4hiJPAiohgJvIgoRgIvIoqRwIuIYsyowJN09aD7EBHDa0YFnu3DBt2HiBheMyrwJN036D5ExPCaUYEXEdGmWTF5gKQlwBKAbdh2wL2JiNlqVuzhZV3aiGjCrAi8iIgmJPAiohgzKvBszx90HyJieM2owIuIaFMCLyKKkcCLiGIk8CKiGAm8iChGAi8iijErhpbFDKN2/05u8fjHt1p/3s23tlp/p6fs3mr9Xz6v/f2Uvb/T3rqxAJr/uPaKP5R1aSMiEngRUY4EXkQUI4EXEcVI4EVEMRJ4EVGMBF5EFCOBFxHF6MuNx5L+J3AfsD2w3PY3xry+GHib7WP60Z+IKFNfR1rYfk8/24uI6NTaIa2kd0q6RdJ3gH3qbRdKelX9+GhJP5S0CnhFW/2IiBjVSuBJOhj4E+BA4MXAIWNe3wb4GPBHwMHArm30IyKiU1t7eEcAn7P9gO17gC+OeX1f4Ce2f2TbwCWbKyZpiaSVklZuYF1LXY6IYTcrrtJmXdqIaEJbgbcceJmkeZK2ozp07fRDYIGkvevnx7XUj4iITVq5Smt7laRPATcAtwPfG/P6Q5KWAF+W9ADwbWC7NvoSETGqtdtSbJ8OnL6Z179KdS4vIqIvZsU5vIiIJiTwIqIYCbyIKEYCLyKKkcCLiGIk8CKiGAm8iChGFuKOqXt4pNXyI7/5Tav127bTJbe3Wn+XHdq/R/+HZxzQav2tfzOntdrrlnUffpo9vIgoRgIvIoqRwIuIYiTwIqIYCbyIKEYCLyKKkcCLiGIk8CKiGAm8iChGAi8iipHAi4hizIqxtPWCP0sAtmHbAfcmImarWbGHl3VpI6IJsyLwIiKaMGMCT9JXJO026H5ExPCaMefwbL940H2IiOE2Y/bwIiLalsCLiGIk8CKiGAm8iChGAi8iipHAi4hiJPAiohgz5j68iGHhDetbrT/yn3e3Wh9g37PvaLX+qZd/rrXab/ps93WNs4cXEcVI4EVEMRJ4EVGMBF5EFCOBFxHFSOBFRDESeBFRjAReRBRjwsCTtEDSg5JW189HJK3u+Fhab79K0sqOr1sk6ar68WJJv5V0vaS1kpZLOqbjvW+V9B+Szmn8XxgRUZvsSIsf2z6wfvxgx+OxnijpRbYvH+e1b9s+BkDSgcDnJT1o+0rbZ0q6C1g0te5HRExe04e0HwTeOdGbbK8G3gec1HD7ERFdTSfw5o05pH1Nx2vfBdZLev4k6qwC9p1Mg5KWSFopaeUG1k2jyxER05s8YHOHtADvB94F/O0EdTTZBm0vA5YBbK8neLJfFxHRqfGrtLa/CcwDnjPBWw8Cbm66/YiIbtq6LeX9wKndXpT0LODdwD+21H5ExGNM55B23ugtKrWv2l7a+QbbX5E0dlKqIyRdD2wL3A6cbPvKabQfETEtUw4823O6bF885vnBHY+vAnaYalsREU2azCHtCLDDmL26Rkl6K3AacE9bbURETLiHZ/vnwO5tdsL2mcCZbbYREZGxtBFRjAReRBQjgRcRxUjgRUQxZM+ukVr1/X0/m8KX7Ay0uchm6g93/X60kfrN1t/T9i7jvTDrAm+qJK203dq0U6k/3PX70Ubq969+DmkjohgJvIgoRgmBtyz1U3+Gt5H6fao/9OfwIiJGlbCHFxEBJPAioiAJvIgoRgIvIoqRwIuIYvwXGk3j7qquoBMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ky1cBbVsCEYO"
      },
      "source": [
        "Attention plots for sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjRW5MOGCRs_"
      },
      "source": [
        "def plot_attention_weights(sentence, translated_tokens, attention_heads):\n",
        "  in_tokens = tf.convert_to_tensor([sentence])\n",
        "  in_tokens = tokenizers.pt.tokenize(in_tokens).to_tensor()\n",
        "  in_tokens = tokenizers.pt.lookup(in_tokens)[0]\n",
        "\n",
        "  fig = plt.figure(figsize=(16, 8))\n",
        "\n",
        "  for h, head in enumerate(attention_heads):\n",
        "    ax = fig.add_subplot(2, 4, h+1)\n",
        "\n",
        "    plot_attention_head(in_tokens, translated_tokens, head)\n",
        "\n",
        "    ax.set_xlabel(f'Head {h+1}')\n",
        "\n",
        "  plt.tight_layout()\n",
        "  plt.show()"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 519
        },
        "id": "mmMa6fr2CUep",
        "outputId": "77988781-ded2-4f1d-ed91-b95f71e6b5c3"
      },
      "source": [
        "plot_attention_weights(sentence, translated_tokens,\n",
        "                       attention_weights['decoder_layer4_block2'][0])"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAInCAYAAADj+u5bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebhkZXnv/e/dcwPNpCiiAioyRVGaVlHBIMag0ZiIUY/oyXGILRqD5o3xxTjk1Sv6epKoCZioOAAeNcYxzsE4EBCICM2oCCaKBmcQBWnoYfd9/qjasHsP3bV3Pc+qvVZ9P9fVV9fUv7p31a7f3vX0WqsiM5EkSZIkSVJ7LRn1AJIkSZIkSRqOCzySJEmSJEkt5wKPJEmSJElSy7nAI0mSJEmS1HIu8EiSJEmSJLWcCzySJEmSJEkt5wKPJEmSJElSy7nAI0mSJEmS1HLLRj2A2iMiVgAH989em5lbRjmPpG6zcyQ1yc6R1DR7R6VFZo56BrVARBwHnA1cDwRwX+B/ZeZ5IxxLUkfZOZKaZOdIapq9oxpc4NFAIuJS4KTMvLZ//mDgnzLzqNFOJqmL7BxJTbJzJDXN3lENHoNHg1o+WT4AmXkdsHyE80jqNjtHUpPsHElNs3dUXGeOwRMRpw1ws1sy8zXVh+mmSyPiPcAH+uefDVwywnmkkbJzqrNzpGnsnarsHGkaO6c6e0fFdWYXrYj4PvC6ndzs1Mw8rIl5uiYiVgJ/DBzTv+h84B8zc9PoppJGx86py86RZrJ36rFzpJnsnLrsHdXQmS14gLdl5tk7ukFE7NXUMF0SEUuBKzLzUOCto55HWiTsnErsHGlO9k4Fdo40JzunEntHtXTpGDxbd3aDzPy7JgbpmsycAK6NiP1HPYu0iNg5ldg50pzsnQrsHGlOdk4l9o5q6dIuWhsyc+2o5+iqiDgPOBK4GLht8vLMfMrIhpJGyM6py86RZrJ36rFzpJnsnLrsHdXQpV20VNdrRz2ApLFi50hqkp0jqWn2jorr0hY8W4GNs10FZGbu3vBIkjrMzpHUNHtHUpPsHKl9urQFz1WZeeSoh+iaiPhaZh4TEbcCU1cDLXaNOzunAjtH2iF7pzA7R9ohO6cCe0c1dWmBRxVk5jH9v9eMehZJ3WfnSGqSnSOpafaOaurSAs9HZ7swIn4b+PPMfHzD83RORBwDPDAzz4yIuwNrMvN7o55LzYiI0wa42S2Z+ZrqwywOdk5lds54s3NmZe9UZOeMNztnVnZOZfbOeKvRO106Bs/xwDuB/YB/Af43cCa9Td3emJmfGOF4rRcRfwmsAw7JzIMjYj/go5n56BGPpoZExPeB1+3kZqdm5mFNzDNqdk5ddo7snJnsnXrsHNk5M9k5ddk7qtE7XdqC5y3AeuAi4In9v0/NzLePdKrueCq9j/HbAJCZP4oINyscL2/LzLN3dIOI2KupYRYBO6cuO0d2zkz2Tj12juycmeycuuwdFe+dLm3Bc9nUg4BFxLWZecgoZ+qSiLg4Mx8eERsyc21E7ApclJlHjHo2aRTsnLrsHGkme6ceO0eayc6py95RDV3agmePiDhxyvllU8+7CeHQPhIR7wL2jIgXAs8H3j3imdSwiHgs8CfA5A/3a4C3Z+a5IxtqdOycuuwc2Tkz2Tv12Dmyc2ayc+qyd1S8d7q0Bc+ZO7g6M/P5jQ3TURHxeOC36e13e05m/tuIR1KDIuJJwNuBN9DblDSAtcBrgJdm5udHOF7j7Jz67JzxZufMZO/UZeeMNztnJjunPntnvNXonc4s8OxIRNwzM3866jm6ICJ2Z8qWX5n5ixGOowZFxLnAyzLzimmXHwGcnpm/OZLBFiE7pxw7Z3zZOfNj75Rh54wvO2d+7Jxy7J3xVaN3urSL1nYiYk/gacBJwGH0jv6uBYqIFwGvB+4AttFbXUzg/qOcS43ad3r5AGTmlRFxz1EMtJjYOWXZOcLO2Sl7pxw7R9g5O2XnlGXviAq906kFnohYDfwevdI5ElgD/D5w3pC59wTeBOyXmU+MiMOBR2bme4ccuU1eATwoM28sGRoR9wFOB46hV2jn01vFvKHk/aiI2xZ4XWfZOVXZObJzZlGjd+wcwM6RnTMrO6cqe0fFe6czu2hFxIeAY4EvAh8GvgL8Z2ber0D2F4AzgVdn5kMiYhlwWWY+eNjstoiIfwVOzMyNhXP/DfgQ8H/6Fz0HeHZmPr7k/Wh4EfFLZv9hHsAxmTlWHx1q59Rl58jOmalW79g5do7snNnYOXXZO6rRO13agudw4GZ6R52+JjMnIqLU6tXdM/MjEfEqgMzcGhEThbLb4lXAhRHxdWDT5IWZecqQuftk5tQDuJ0VES8fMpOIOAB4YGZ+qf8/D8sy89Zhc8fc7+3gur9tbIrFw86py86RnTNTrd6xc+wc2TmzsXPqsndUvHc6s8CTmQ+NiEOBZwFfiogbgTWFDgB2W0Tcjd4mbkTE0cCvhsxsm3fRW7W/it4+oqXcFBHPAf6pf/5ZwE3DBPY/ZnA9sDfwAOA+wDuBxw2TO+4y899HPcNiYudUZ+eMOTtnpoq9Y+fYOWPPzpnJzqnO3hlzNXqnS7toHZ2Z/zHl/FH0vpmfAdyQmY8aInstvf0YHwRcDewDPH22AyJ1VURclplHVsg9gN5j+0h6JX8hcEpm/mCIzMuBhwNfn5w5Iq4at80+S4uIr9L/QTyLzMyxKng7py47R3bOTLV6x86xc2TnzMbOqcveUY3e6dICz4bMXDvL5QEcm5nDHAhsJTABHEJvf7hrgSWZuWmH/7BDIuJNwPXAZ9h+E8IFf4xfRCwF3p+Zzx56wO1zv56Zj5gszf5+vRsy84iS9zNu+j/UpzsaeCXws8x8WMMjjZSdU5edIztnplq9Y+fYObJzZmPn1GXvqEbvdGYXrblkbwVrqE+0AS7ql9s3Jy+IiA3AjMLrsGf1/37VlMuG+hi//n68B0TEiszcPNR02/v3iPgLYHVEPB54Cb3i1BAy89LJ0xHxm8BrgVXAyZn5hZENtsjYOcXYOWPOzhlcgd6xc+ycsWfnDM7OKcbeGXM1eqdLCzz3j4hPz3VlZj5lvoERsS9wb3rfyEfSW2EG2B3YZUFTtlQW+GSgOXwXuKD/3N35UXCZ+dYhMk8FXkBvf9YXAZ8H3jPMkOqJiBOA19D7X4Y3ZuZXRzzSKNk5Fdk5AjtnFkV7x865i50jsHNmYedUZO8IyvdOlxZ4fg68pXDmCcBz6R1E6i3cVUK3An9R+L4WpYg4PjO/EhEnznZ9Zn5iyLv4r/6fJcCaIbMAyMxtwLv7f1RIRHyD3j7SfwNc1L/szv9pycwNIxptVOycCuwcTbJzZlW6d+wcO0d9ds6s7JwK7B1NqtE7XToGT5WDVPWzn5aZH6+RvdhFxOsz8y8j4sxZrs7MfP6Q+WtL/8CMiO8xy8GqMnPBmzv2v/7ZMof9+o8FLszMiSmXFX9MSoiIc9nxQcCOb3CckbNz6rBztsst3jt2TrvV6h07x87p59o5ds527Jw67J07M31/VaF3urQFz/cqZt8nInant7r8bnr7h56amV+seJ+LQr98lgBfyMyPVLiLt/Q31/wY8M+ZeXWBzHVTTq8Cnk7vI/2G8dlpmU8FfjRkJsA5wDci4umZ+bP+Ze9hEe6DnJnHjXqGRcbOqcDO2U6N3rFz2q1W79g5dg7YOceNeoZFyM6pwN65k++vavROZnbiD/AwYN8p5/8Q+BRwGrD3kNlX9P8+Afgk8Bv0jho+8q+7wcf3korZ+wKnABfQ26/zNRXu49LCeUvorQwPm3MZ8GTgGuBRk5eN+vmeY9ZXTjn99GnXvWnU843g8bBz6j6+ds7MzKF7x85p959avWPn2DlzZNo5d523c9LOqfD42jvb5/n+avvrFtQ7S+iOdwGbASLiMcCbgfcDvwLOGDJ7cv/QJ9H72LlvTrlsXHwpIl4REfeNiL0n/5QIzsyfZOZpwMnA5cDrhsmLiLVT/qyLiJMpv7XaA4F7FMjJzPws8BTg7RHxUubeTG/U/seU06+adt0TmhxkkbBz6rJzZirRO3ZOu9XqHTvHzpmNnXMXO8fOqcHe2Z7vr7a3oN7p0i5aSzPzF/3TzwTOyN6+nR+PiMuHzL40Is6h95F1p0bEGmDbkJlt80x6L4yXTLt82P29D+tnPw24Cfhn4M+GyaR30LbJF/FW4Hp6mxEuWETcyvbF8BPglcNkTkYDZOZ3+j843wccUSC3hpjj9Gznx4GdU9dYdw5U6x07p91q9Y6dY+fYOXbObOycusa6d3x/BVTonU4t8ETEsszcCjwOWD/lumG/zhfQ++iyb2XmxojYH3j5MIEREcCzgftn5hv6mftm5sVDzkpEPAQ4tn/2/My8YthM4HB65XMMvRfi+cA7C+S+j17pnJCZJfa5hN7+nMldL4oEntx7yBf8EYF70Hu+7jf1+Rp20Jxy4LrM/DXwjH72YpRznJ7t/Diwc+7KtnPKdw5U6B07p/Vq9U7xzoF6vWPn2DmV2Dkz2Tl3Zds7vr+qoXjvdOlTtF4N/A5wI7A/sDYzMyIOAs7OzEcPkf0OeqvKx2fmYRGxF/DFzHzYYsrs574MeCEw+fF6T6W32n76kLkfAW4BPti/6CRgj8x8xjC5NUTEh+jtM/wpeiX0u8DFwHcAMvP1C8gs+nxFxCsz868j4nRmP3r8KQvJrSkiJoDb6D2mq4GNk1cBqzJz+ahmGwU7585cO6dC5/Rziz1ndk431Oqdiv1Qo8vsHDunCjtnJjvnzlx7x/dXVdTonc4s8ABExNHAveh9Y9zWv+xgYLcc4mPRImJDZq6NKR8VGBFXZOZDFlNmP+NK4JFTvv5dgYsyc6jN0iLiW5l5+M4um0feRzLzGRFxFdu/AIPefpMLnjcizgOelJm39s+vAT6XmY8ZIrPo8xURN2Xm3SLi5cDN06/PzLMXOquaY+fYOf3s4p3Tzyn2nNk53VGjdyr2Q40us3PsHDVo3Dunn2Hv+P6qNTqzi9bkN8j0yzPzup3dZgBbImIp/RdKROzD8PuJ1siE3gt4Ysr5CSiy3/CGiDg6M/8DICIeAVwyRN7L+n8/eejJZron/QPC9W3uXzaM0s/XTyNiP+B5wHGUeY6qGuT1M8RrrHXsnDvZOXU6B8o+Z3ZOB1TsnVr9UCPXzrFzqrBzZrJz7mTv+P6qihq905kFHuCw/urqXILefn4LcRq9j/C7R0S8EfgDevuNDqNGJsCZwNcj4pP9878PvLdA7lHAhRHxg/75/YFrJ1eI57sinJk/7v/9/QKzTfd+4OJpj8FZQ2aWfr7eAXyZ3kHULp1yedAruaEOrjabiNg3M38yRETN11gb2Tk9dk6dzoGyz1njnQND946dM1Otx6RWP9TItXPsnDnZOcXZOT32ju+vZrUY3191ZhetiDhggJtNZOYNC8w/lN7BxQL4cmZes5Cc2pn93LX0DtYFvYOAXVYgc4eP73yLJGYeNf3Oq3pxuft88mbJX8tdB0I7r9BjUON74B2Z+eJhcwa8r89l5pOG+PdVX2NtY+dsl2vnVOicfm7R56zJzunf34J7x86ZqeZjUrEfanSZnWPnzHV/dk5Bds52ufaO769mu69F9/6qMws8kiRJkiRJ42rJqAeQJEmSJEnScDq7wBMR68c9t02z1spt06y1cts0a9v5/PkY1Mpt06y1cu2cmXz+2pXbpllr5Tpru7Xp+auV26ZZa+U66+LO7ewCD1CrlNuU26ZZa+W2adZauW2ate18/nwMauW2adZauXbOTD5/7cpt06y1cp213dr0/NXKbdOstXKddRHndnmBR5IkSZIkaSy07iDLK5auztXLdn4A8M0Tt7Ni6eqBc+91yC8Hut0vb9rGnncbbF3sx1fvNvD9b2YTK1g50G0Hfc62sInlA2bOR5ty2zRrrdzFMOsd3Mbm3BTFh2jAoJ0D8++d0pm5ecvAuYvh+2LbA1cMlvmr21m+x+CP65IfDtbRW7ZuZPmyXQYL3XjHwPff1dfxqHPHpnNiVa6KXQe67Za8g+WxaqDbHvzg2wae4ec3TbDP3Zbu9HbXXTng64d5fk/M45nbkptYHgPmzuNX3lF/D3c1t6uztrtzVuYqBuyceTwmBx+xceAZanQOjP77oqu5zro4cm/l5hszc5/ply8rPlVlq5ftzqP2e3bx3L/49KeKZ/7V4Y8qngmw7Y7B32hIi8HX88ujHmHBVi/bnUfte1L54CXlfw/c+oN2fXLrr99+vyq5a15ddpENIC/7dvFMALZN1Mkdc23unFWxK0cvf0Lx3H895+LimSfc+8jimQCxdOdv9BYit26tkltFVForaNl/7LZFqzuHXXnEkt8qnnvOOUN/gvYMtTrH14Xa6Ev5se/Pdrm7aEmSJEmSJLWcCzySJEmSJEkt5wKPJEmSJElSy7nAI0mSJEmS1HIu8EiSJEmSJLWcCzySJEmSJEktV2yBJyL2jIiX9E8fFxGfneN274mIw0vdr6TxZe9IapKdI6lJdo6k+Sq5Bc+ewEt2dqPM/KPM/FbB+5U0vuwdSU2ycyQ1yc6RNC8lF3jeDDwgIi4H/gbYLSI+FhHfjogPRkQARMS5EbEuIpZGxFkRcXVEXBURf1pwFknjwd6R1CQ7R1KT7BxJ87KsYNapwIMy86ERcRzwKeA3gB8BFwCPBr425fYPBe6dmQ+C3iaIcwVHxHpgPcCqpWsKjiyp5ar0jp0jaQ71O4ddqg0vqXXsHEnzUvMgyxdn5g2ZuQ24HDhw2vXfBe4fEadHxBOAW+YKyswzMnNdZq5bsXR1vYkltV2R3rFzJA2oeOcsj1V1J5bUZuU7h5V1J5bUqJoLPJumnJ5g2tZCmXkz8BDgXOBk4D0VZ5E0HuwdSU2ycyQ1yc6RtEMld9G6FRh4X4aIuDuwOTM/HhHXAh8oOIuk8WDvSGqSnSOpSXaOpHkptsCTmTdFxAURcTVwO/DTnfyTewNnRsTkVkSvKjWLpPFg70hqkp0jqUl2jqT5KrkFD5l50hyXv3TK6eOmXLW25P1LGj/2jqQm2TmSmmTnSJqPmsfgkSRJkiRJUgNc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeWKfopWm536Zy8unvnTs24vnglwv2ddVSW3lmX3vEfxzK0/2dmnREpl5PJlbL333uWDL/5m8cglu+xSPBNg2+13VMld+s67V8m9dn0Wzzz4RRPFM6XZxJIlLNlt1+K5Tzjg4cUzP/iDrxbPBHj2fR9dJbeaJUvLZ26zc9SMWL6MZfvcs3juY5/3R8Uzlz9yS/FMgKUbrq2Sy6H3rxL7w+P3LJ55r7deWDxTo+EWPJIkSZIkSS3nAo8kSZIkSVLLucAjSZIkSZLUci7wSJIkSZIktZwLPJIkSZIkSS3nAo8kSZIkSVLLjWSBJyL8HDZJjbJ3JDXJzpHUJDtHEoxogSczHzWK+5U0vuwdSU2ycyQ1yc6RBKPbgufX/b/vFRHnRcTlEXF1RBw7inkkdZ+9I6lJdo6kJtk5kgCWjfj+TwLOycw3RsRSYJcRzyOp++wdSU2ycyQ1yc6RxtioF3i+AbwvIpYD/5KZl892o4hYD6wHWLV0TYPjSeqgnfbOdp2zYo+Gx5PUMfPrnCW7NTyepI6ZX+cstXOkLhnpp2hl5nnAY4AfAmdFxB/OcbszMnNdZq5bsXR1ozNK6pZBemdq5yxfvmvjM0rqjvl2zopY1fiMkrpj3p2zxPdWUpeMdIEnIg4AfpqZ7wbeA6wd5TySus/ekdQkO0dSk+wcabyNehet44A/j4gtwK+BWbfgkaSCjsPekdSc47BzJDXnOOwcaWyNZIEnM3fr/302cPYoZpA0XuwdSU2ycyQ1yc6RBCPeRUuSJEmSJEnDc4FHkiRJkiSp5VzgkSRJkiRJajkXeCRJkiRJklrOBR5JkiRJkqSWG/XHpC9AQETx1F0/e1nxzIPO2614JkDuVif3O+88qErufd+/tHjmyhtvKp4JkBMTVXLJrJOr+m67Hb5+1ainGMi2224b9QjzstvLbqiSu++796+SKzVi+TK41z7lc6+7tXjkcw46vngmwAuu+3aV3Pcecv8queS28pkVftcF/H1EM+SWrWz9yU+L5+6ypPx2BLl5S/FMgIlKuUtvKP+4Ahx30neKZ173DyuLZ0Lv+6uKbZXes3WAW/BIkiRJkiS1nAs8kiRJkiRJLecCjyRJkiRJUsu5wCNJkiRJktRyLvBIkiRJkiS1nAs8kiRJkiRJLecCjyRJkiRJUssVX+CJiD0j4iX908dFxGdL34ckTWXvSGqSnSOpSXaOpEHV2IJnT+AlFXIlaS72jqQm2TmSmmTnSBrIsgqZbwYeEBGXA1uA2yLiY8CDgEuB52RmRsRRwFuB3YAbgedm5o8rzCOp++wdSU2ycyQ1yc6RNJAaW/CcCvxXZj4U+HPgSODlwOHA/YFHR8Ry4HTgDzLzKOB9wBvnCoyI9RFxSURcsnliY4WRJbVc0d6Z2jlb2NTIFyCpVap1jr/nSJqFv+dIGkiNLXimuzgzbwDorzofCPyS3orzv0UEwFJgztXlzDwDOANgj5X7ZuV5JbXfUL0ztXN2j73tHEk7U6xz9lh9LztH0s74e46kWTWxwDN1WXiif58BfDMzH9nA/UsaP/aOpCbZOZKaZOdImlWNXbRuBdbs5DbXAvtExCMBImJ5RPxGhVkkjQd7R1KT7BxJTbJzJA2k+BY8mXlTRFwQEVcDtwM/neU2myPiD4DTImKP/hx/B3yz9DySus/ekdQkO0dSk+wcSYOqsotWZp40x+UvnXL6cuAxNe5f0vixdyQ1yc6R1CQ7R9IgauyiJUmSJEmSpAa5wCNJkiRJktRyLvBIkiRJkiS1nAs8kiRJkiRJLecCjyRJkiRJUstV+RStmnLzZrZ+/78rBGfxyImbf1U8E4BtE1Viz37ke6vkvum1JxbP3Lp1a/FMaU4V+kHw+UM+XyX3hA8fVSVXakJu2sy271xfPrfCz80onthz9gnHVcl93rfPq5L7/kcdWTxz4qZfFM+U5lTh95ytP/px8cxYurR4JkAsqdNm33vHflVyf/y+Q4pn3mPThcUzAVhS5znT3NyCR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJartgCT0ScEhHXRMTNEXHqPP7dgRFxUqk5JI0HO0dS0+wdSU2ycyTN17KCWS8Bfiszb5jtyohYlplbZ7nqQOAk4EMFZ5HUfXaOpKbZO5KaZOdImpciCzwR8U7g/sAXIuJ9wAMy86URcRZwB3AkcEFEfAr4+/4/S+AxwJuBwyLicuDszHxbiZkkdZedI6lp9o6kJtk5khaiyAJPZp4cEU8AHgs8edrV9wEelZkTEfEZ4I8z84KI2I1eOZ0KvCIzp/+7O0XEemA9wCp2KTGypBazcyQ1rWbv2DmSprNzJC1EEwdZ/mhmTvRPXwC8NSJOAfacY5PCGTLzjMxcl5nrlrOy2qCSOsHOkdS0oXpnu86JVVUHldQJ5TrH33OkTmligee2yROZ+Wbgj4DV9DYpPLSB+5c0XuwcSU2zdyQ1yc6RNKuSB1neqYh4QGZeBVwVEQ8DDgX+G1jT5BySxoOdI6lp9o6kJtk5kqZqYgueqV4eEVdHxJXAFuALwJXARERcERF/2vA8krrNzpHUNHtHUpPsHEl3KrYFT2Ye2D95Vv8Pmfncabf5kzn++fGl5pA0HuwcSU2zdyQ1yc6RNF9Nb8EjSZIkSZKkwlzgkSRJkiRJajkXeCRJkiRJklrOBR5JkiRJkqSWc4FHkiRJkiSp5Yp9ilajMkc9wWC2TYx6gnl5wwOOqpJ7zg//pXjmE/ZfVzwTILdurZKrFguIZeWr0u81ePTLT66Su+ZL/10+9ISfl88EcsvmKrlqr1iyhCW7ri6eO/HL8t9rOVHn95yt3/t+ldyz1z2oSu7fX/2Z4pmnHPTY4plg56hBFd6vte13pwOec12V3E/81/nFM5/6jw8vnglAbquTqzm5BY8kSZIkSVLLucAjSZIkSZLUci7wSJIkSZIktZwLPJIkSZIkSS3nAo8kSZIkSVLLucAjSZIkSZLUcgta4ImIAyPi6mHvPCKuj4i7D5sjqfvsHUlNsnMkNcnOkVSCW/BIkiRJkiS13DALPMsi4oMRcU1EfCwidomIx0XEZRFxVUS8LyJWAsx1+aSIWB0RX4iIFw711UjqOntHUpPsHElNsnMkDWWYBZ5DgH/MzMOAW4D/BzgLeGZmPhhYBrw4IlbNdvmUnN2AzwD/lJnvHmIeSd1n70hqkp0jqUl2jqShDLPA89+ZeUH/9AeAxwHfy8zr+pedDTyGXlHNdvmkTwFnZub757qjiFgfEZdExCVb2DTEyJJarpHe2a5z0s6RxljjnbM5by/+RUhqjeZ/z/G9ldQpwyzw5LTzv1xgzgXAEyIi5ryjzDMyc11mrlvOyrluJqn7Gumd7Ton7BxpjDXeOSti9QLvQlIHNP97ju+tpE4ZZoFn/4h4ZP/0ScAlwIERcVD/sv8J/Dtw7RyXT3odcDPwD0PMImk82DuSmmTnSGqSnSNpKMMs8FwL/HFEXAPsBbwNeB7w0Yi4CtgGvDMz75jt8mlZLwNWR8RfDzGPpO6zdyQ1yc6R1CQ7R9JQli3kH2Xm9cChs1z1ZeDIWW4/1+UHTjn7vIXMImk82DuSmmTnSGqSnSOphGG24JEkSZIkSdIi4AKPJEmSJElSy7nAI0mSJEmS1HIu8EiSJEmSJLWcCzySJEmSJEktt6BP0ZLm4wn7ryueufn4hxbPBLj+95ZWyT3g0xPFM1d+5crimQCxYnn5zI3tXUuOJUtZsmZN8dyJm28untk2e3z5uiq5275zr+KZE4/Yp3gmwIobflEl9/aDys+7bOPW4pkAy35+a/HM+P75xTObkhMTTPzqllGPMZjMUU8wL9tuLf+9BvAnBzy6eOZZP/hK8UyAFxz11Cq5EzdV6LJt5X930hwiKmRW+N2vZd8TuWlTldwT73dM8cytxz+4eCbA9b9b/qTTiQoAACAASURBVH0FwL0uKP/zZ9ePfb14JgBL6ry/ZI6XQ3vfdUmSJEmSJAlwgUeSJEmSJKn1XOCRJEmSJElqORd4JEmSJEmSWs4FHkmSJEmSpJZzgUeSJEmSJKnlXOCRJEmSJElquUW1wBMRF456Bknjw86R1CQ7R1LT7B1pvCyqBZ7MfNSoZ5A0PuwcSU2ycyQ1zd6RxsuiWuCJiF+PegZJ48POkdQkO0dS0+wdabwsG/UAg4iI9cB6gFXsMuJpJHXddp2zZLcRTyOp6/w9R1KT7BypuxbVFjxzycwzMnNdZq5bzspRjyOp46Z2zopYNepxJHWcv+dIapKdI3VXKxZ4JEmSJEmSNDcXeCRJkiRJklrOBR5JkiRJkqSWW1QLPJnp0UwlNcbOkdQkO0dS0+wdabwsqgUeSZIkSZIkzZ8LPJIkSZIkSS3nAo8kSZIkSVLLucAjSZIkSZLUci7wSJIkSZIktdyyUQ+gRSTqrPct2Wuv4pmrr/lJ8UyAu937vlVyf/ib5R/bB3xtefFMgNht1/Khd7R3LTknJpi4+eZRj9FJEzf9ok5whdwlEcUzAXKP3avknvW+DxXPfOEBxxbPBJiokJnbNldIbVDmqCfQiL3gqKdWyf3hcx5YJffWh2wqnnnIS64pngmw7fbby4e2/SVbo3OyRrsLILeU/xm3/PyrimcCHHJFnQ9xe/U3vlQ88w2feFjxTIAlK+q8Z2OOKmvvuy5JkiRJkiQBLvBIkiRJkiS1ngs8kiRJkiRJLecCjyRJkiRJUsu5wCNJkiRJktRyLvBIkiRJkiS1nAs8kiRJkiRJLbesiTuJiP8P+DWwO3BeZn5p2vXHAa/IzCc3MY+k7rN3JDXJzpHUJDtH0mwaWeCZlJmva/L+JMnekdQkO0dSk+wcSVNV20UrIl4dEddFxNeAQ/qXnRURf9A//YSI+HZEbABOrDWHpPFh70hqkp0jqUl2jqSdqbLAExFHAf8DeCjwO8DDpl2/Cng38LvAUcC+O8lbHxGXRMQlW9hUY2RJLVeyd+wcSTtj50hqkp0jaRC1tuA5FvhkZm7MzFuAT0+7/lDge5n5ncxM4AM7CsvMMzJzXWauW87KSiNLarlivWPnSBqAnSOpSXaOpJ3yU7QkSZIkSZJartYCz3nA70fE6ohYQ29Twam+DRwYEQ/on39WpTkkjQ97R1KT7BxJTbJzJO1UlU/RyswNEfHPwBXAz4BvTLv+johYD3wuIjYC5wNraswiaTzYO5KaZOdIapKdI2kQ1T4mPTPfCLxxB9f/K719RSWpCHtHUpPsHElNsnMk7YzH4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWq5ap+ipRbaNlElduLnP6+SW8PdPvCzKrn77FH+Uyq//eYHFc8EWPnzpcUzN52xsnim2m/pXntVyZ341S3FM2NJFM8EyDs2Vcldf+LJxTO/c/aK4pkAD3h3lg/dcEH5TKlBtX532u+MjVVyc1P5Lnvfd88tngnw/N94YvHM+LX/Z64GLSn/uzqVfs+pMivwh59+SfHMQx54Y/FMgJsevk+VXN7/gVkvto0kSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJabtmoBxhERKwH1gOsYpcRTyOp6+wcSU2ycyQ1yc6RuqsVW/Bk5hmZuS4z1y1n5ajHkdRxdo6kJtk5kppk50jd1YoFHkmSJEmSJM3NBR5JkiRJkqSWWzQLPBHx+YjYb9RzSBof9o6kJtk5kppk50jjZ9EcZDkzf2fUM0gaL/aOpCbZOZKaZOdI42fRbMEjSZIkSZKkhXGBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklls0n6IlLQa5ZXOV3Ilf/LJ45qGn31g8E+CVX/hk8cwXf/znxTPVfhM33zzqEQaW2yrlTkxUyY3Lv10885C/Pah4JsCzPvKl4pnfPfFXxTOlLti2cWOV3Fi2vHjm8048uXgmwHMv+VzxzO+ceHvxzNaLKJ+ZWT6zjbaV/90htxSP7KkwK8A9Dqnw3uLHdd6vrLp57yq5c3ELHkmSJEmSpJZzgUeSJEmSJKnlXOCRJEmSJElqORd4JEmSJEmSWs4FHkmSJEmSpJZzgUeSJEmSJKnlXOCRJEmSJElqORd4JEmSJEmSWm6nCzwRcWBE3B4Rl/fPT0TE5VP+nNq//NyIuGTKv1sXEef2Tx8XEb+KiMsi4tqIOC8injzltn8aET+IiLcX/woltYqdI6lp9o6kJtk5kmpZNuDt/iszH9o/ffuU09PdIyKemJlfmOW68zPzyQAR8VDgXyLi9sz8cma+LSJuBtbNb3xJHWXnSGqavSOpSXaOpOJK76L1N8Crd3ajzLwceAPw0kFCI2J9RFwSEZdsYdOQI0rqEDtHUtOK946dI2kH7BxJA1vIAs/qaZsQPnPKdRcBmyPisQPkbAAOHeQOM/OMzFyXmeuWs3IBI0tqMTtHUtMa7R07Rxp7do6kIgbdRWuqHW1CCPBXwGuA/3cnObGA+5Y0fuwcSU2zdyQ1yc6RVETxT9HKzK8Aq4Gjd3LTI4FrSt+/pPFi50hqmr0jqUl2jqRB1fqY9L8CXjnXlRFxBPBa4B8q3b+k8WLnSGqavSOpSXaOpJ1ayC5aqyc/0q/vXzPz1Kk3yMzPR8TPp/27YyPiMmAX4GfAKZn55QXcv6TxYudIapq9I6lJdo6kIua9wJOZS+e4/Lhp54+acvpcYI/53pck2TmSmmbvSGqSnSOplEF20ZoA9pi2qlxURPwp8Crgllr3Iak17BxJTbN3JDXJzpFUxU634MnM/wbuW3OIzHwb8Laa9yGpHewcSU2zdyQ1yc6RVEutgyxLkiRJkiSpIS7wSJIkSZIktVxk5qhnmJf+0eO/P8BN7w7cWGGENuW2adZauW2atVbuYpj1gMzcp8IM1c2jc6C7z9+oc9s0a63cNs1aK9fOmamrz19Xc9s0a63crs5q5wynTbltmrVWrrMujtxZe6d1CzyDiohLMnPdOOe2adZauW2atVZum2ZtO58/H4NauW2atVaunTOTz1+7cts0a61cZ223Nj1/tXLbNGutXGdd3LnuoiVJkiRJktRyLvBIkiRJkiS1XJcXeM4wt1Wz1spt06y1cts0a9v5/PkY1Mpt06y1cu2cmXz+2pXbpllr5Tpru7Xp+auV26ZZa+U66yLO7ewxeNSsiPh1Zu425fxzgXWZ+dIC2ecCr8jMS6Zd/lLg5cADgH0ys8aBriQtQiPqnA8C64AtwMXAizJzy7D3J2nxG1HnvJde5wRwHfDczPz1sPcnqR1G0TtTrj8NeP7U+1c7dHkLHnXfBcBvMfiR/yVpGB8EDgUeDKwG/mi040jquD/NzIdk5hHAD4Ch39RJ0s5ExDpgr1HPoYVxgUfVRcQ+EfHxiPhG/8+j+5c/PCIuiojLIuLCiDikf/nqiPhwRFwTEZ+k90Zqhsy8LDOvb+4rkdQGFTvn89lHbwue+zT2RUlatCp2zi3920f/Nm52Lwmo1zsRsRT4G+CVjX0xKmrZqAdQZ6yOiMunnN8b+HT/9N8Db8vMr0XE/sA5wGHAt4FjM3NrRPwW8CbgacCLgY2ZeVhEHAFsaOyrkNQWI+uciFgO/E/gZUW/IkmL2Ug6JyLOBH4H+BbwZ6W/KEmL2ih656XApzPzx721ZbWNCzwq5fbMfOjkmcl9RPtnfws4fEpJ7B4RuwF7AGdHxAPp/a/U8v71jwFOA8jMKyPiyvrjS2qZUXbOPwLnZeb5Jb4QSa0wks7JzOf1/0f9dOCZwJnFviJJi12jvRMR+wFPB44r/pWoMS7wqAlLgKMz846pF0bE24GvZuZTI+JA4NzmR5PUQdU6JyL+EtgHeNHwY0rqiKq/52TmRER8mN4uEy7wSII6vXMkcBDwn/2Fo10i4j8z86AiE6sRHoNHTfgi8CeTZyJiciV6D+CH/dPPnXL784CT+rd9EHBE/REldUiVzomIPwJOAJ6VmdvKjiypxYp3TvQcNHkaeAq9XS8kCSr0TmZ+LjP3zcwDM/NAert0ubjTMi7wqAmnAOsi4sqI+BZwcv/yvwb+/4i4jO23JnsHsFtEXAO8Abh0ttCIOCUibqB3oNMrI+I91b4CSW1SpXOAdwL3BC6KiMsj4nV1xpfUMjU6J+jtZnEVcBVwr/5tJQnq/a6jloveh4FIkiRJkiSprdyCR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeVc4JEkSZIkSWo5F3gkSZIkSZJazgUeSZIkSZKklnOBR5IkSZIkqeWWjXoAtUdErAAO7p+9NjO3jHIeSd1m50hqkp0jqWn2jkqLzBz1DGqBiDgOOBu4HgjgvsD/yszzRjiWpI6ycyQ1yc6R1DR7RzW4wKOBRMSlwEmZeW3//MHAP2XmUaOdTFIX2TmSmmTnSGqavaMaPAaPBrV8snwAMvM6YPkI55HUbXaOpCbZOZKaZu+ouM4cgyciThvgZrdk5muqD9NNl0bEe4AP9M8/G7hkhPNII2XnVGfnSNPYO1XZOdI0dk519o6K68wuWhHxfeB1O7nZqZl5WBPzdE1ErAT+GDimf9H5wD9m5qbRTSWNjp1Tl50jzWTv1GPnSDPZOXXZO6qhM1vwAG/LzLN3dIOI2KupYbokIpYCV2TmocBbRz2PtEjYOZXYOdKc7J0K7BxpTnZOJfaOaunSMXi27uwGmfl3TQzSNZk5AVwbEfuPehZpEbFzKrFzpDnZOxXYOdKc7JxK7B3V0qVdtDZk5tpRz9FVEXEecCRwMXDb5OWZ+ZSRDSWNkJ1Tl50jzWTv1GPnSDPZOXXZO6qhS7toqa7XjnoASWPFzpHUJDtHUtPsHRXXpS14tgIbZ7sKyMzcveGRJHWYnSOpafaOpCbZOVL7dGkLnqsy88hRD9E1EfG1zDwmIm4Fpq4GWuwad3ZOBXaOtEP2TmF2jrRDdk4F9o5q6tICjyrIzGP6f68Z9SySus/OkdQkO0dS0+wd1dSlBZ6PznZhRPw28OeZ+fiG5+mciDgGeGBmnhkRdwfWZOb3Rj2XmhERpw1ws1sy8zXVh1kc7JzK7JzxZufMyt6pyM4Zb3bOrOycyuyd8Vajd7p0DJ7jgXcC+wH/Avxv4Ex6m7q9MTM/McLxWi8i/hJYBxySmQdHxH7ARzPz0SMeTQ2JiO8Dr9vJzU7NzMOamGfU7Jy67BzZOTPZO/XYObJzZrJz6rJ3VKN3urQFz1uA9cBFwBP7f5+amW8f6VTd8VR6H+O3ASAzfxQRblY4Xt6WmWfv6AYRsVdTwywCdk5ddo7snJnsnXrsHNk5M9k5ddk7Kt47XdqC57KpBwGLiGsz85BRztQlEXFxZj48IjZk5tqI2BW4KDOPGPVs0ijYOXXZOdJM9k49do40k51Tl72jGrq0Bc8eEXHilPPLpp53E8KhfSQi3gXsGREvBJ4PvHvEM6lhEfFY4E+AyR/u1wBvz8xzRzbU6Ng5ddk5snNmsnfqsXNk58xk59Rl76h473RpC54zd3B1ZubzGxumoyLi8cBv09vv9pzM/LcRj6QGRcSTgLcDb6C3KWkAa4HXAC/NzM+PcLzG2Tn12Tnjzc6Zyd6py84Zb3bOTHZOffbOeKvRO51Z4NmRiLhnZv501HN0QUTszpQtvzLzFyMcRw2KiHOBl2XmFdMuPwI4PTN/cySDLUJ2Tjl2zviyc+bH3inDzhlfds782Dnl2Dvjq0bvdGkXre1ExJ7A04CTgMPoHf1dCxQRLwJeD9wBbKO3upjA/Uc5lxq17/TyAcjMKyPinqMYaDGxc8qyc4Sds1P2Tjl2jrBzdsrOKcveERV6p1MLPBGxGvg9eqVzJLAG+H3gvCFz7wm8CdgvM58YEYcDj8zM9w45cpu8AnhQZt5YMjQi7gOcDhxDr9DOp7eKeUPJ+1ERty3wus6yc6qyc2TnzKJG79g5gJ0jO2dWdk5V9o6K905ndtGKiA8BxwJfBD4MfAX4z8y8X4HsLwBnAq/OzIdExDLgssx88LDZbRER/wqcmJkbC+f+G/Ah4P/0L3oO8OzMfHzJ+9HwIuKXzP7DPIBjMnOsPjrUzqnLzpGdM1Ot3rFz7BzZObOxc+qyd1Sjd7q0Bc/hwM30jjp9TWZORESp1au7Z+ZHIuJVAJm5NSImCmW3xauACyPi68CmyQsz85Qhc/fJzKkHcDsrIl4+ZCYRcQDwwMz8Uv9/HpZl5q3D5o6539vBdX/b2BSLh51Tl50jO2emWr1j59g5snNmY+fUZe+oeO90ZoEnMx8aEYcCzwK+FBE3AmsKHQDstoi4G71N3IiIo4FfDZnZNu+it2p/Fb19REu5KSKeA/xT//yzgJuGCex/zOB6YG/gAcB9gHcCjxsmd9xl5r+PeobFxM6pzs4Zc3bOTBV7x86xc8aenTOTnVOdvTPmavROl3bROjoz/2PK+aPofTM/A7ghMx81RPZaevsxPgi4GtgHePpsB0Tqqoi4LDOPrJB7AL3H9pH0Sv5C4JTM/MEQmZcDDwe+PjlzRFw1bpt9lhYRX6X/g3gWmZljVfB2Tl12juycmWr1jp1j58jOmY2dU5e9oxq906UFng2ZuXaWywM4NjOHORDYSmACOITe/nDXAksyc9MO/2GHRMSbgOuBz7D9JoQL/hi/iFgKvD8znz30gNvnfj0zHzFZmv39ejdk5hEl72fc9H+oT3c08ErgZ5n5sIZHGik7py47R3bOTLV6x86xc2TnzMbOqcveUY3e6cwuWnPJ3grWUJ9oA1zUL7dvTl4QERuAGYXXYc/q//2qKZcN9TF+/f14D4iIFZm5eajptvfvEfEXwOqIeDzwEnrFqSFk5qWTpyPiN4HXAquAkzPzCyMbbJGxc4qxc8acnTO4Ar1j59g5Y8/OGZydU4y9M+Zq9E6XFnjuHxGfnuvKzHzKfAMjYl/g3vS+kY+kt8IMsDuwy4KmbKks8MlAc/gucEH/ubvzo+Ay861DZJ4KvIDe/qwvAj4PvGeYIdUTEScAr6H3vwxvzMyvjnikUbJzKrJzBHbOLIr2jp1zFztHYOfMws6pyN4RlO+dLi3w/Bx4S+HME4Dn0juI1Fu4q4RuBf6i8H0tShFxfGZ+JSJOnO36zPzEkHfxX/0/S4A1Q2YBkJnbgHf3/6iQiPgGvX2k/wa4qH/Znf/TkpkbRjTaqNg5Fdg5mmTnzKp079g5do767JxZ2TkV2DuaVKN3unQMnioHqepnPy0zP14je7GLiNdn5l9GxJmzXJ2Z+fwh89eW/oEZEd9jloNVZeaCN3fsf/2zZQ779R8LXJiZE1MuK/6YlBAR57Ljg4Ad3+A4I2fn1GHnbJdbvHfsnHar1Tt2jp3Tz7Vz7Jzt2Dl12Dt3Zvr+qkLvdGkLnu9VzL5PROxOb3X53fT2Dz01M79Y8T4XhX75LAG+kJkfqXAXb+lvrvkx4J8z8+oCmeumnF4FPJ3eR/oN47PTMp8K/GjITIBzgG9ExNMz82f9y97DItwHOTOPG/UMi4ydU4Gds50avWPntFut3rFz7Bywc44b9QyLkJ1Tgb1zJ99f1eidzOzEH+BhwL5Tzv8h8CngNGDvIbOv6P99AvBJ4DfoHTV85F93g4/vJRWz9wVOAS6gt1/nayrcx6WF85bQWxkeNucy4MnANcCjJi8b9fM9x6yvnHL66dOue9Oo5xvB42Hn1H187ZyZmUP3jp3T7j+1esfOsXPmyLRz7jpv56SdU+HxtXe2z/P91fbXLah3ltAd7wI2A0TEY4A3A+8HfgWcMWT25P6hT6L3sXPfnHLZuPhSRLwiIu4bEXtP/ikRnJk/yczTgJOBy4HXDZMXEWun/FkXESdTfmu1BwL3KJCTmflZ4CnA2yPipcy9md6o/Y8pp1817bonNDnIImHn1GXnzFSid+ycdqvVO3aOnTMbO+cudo6dU4O9sz3fX21vQb3TpV20lmbmL/qnnwmckb19Oz8eEZcPmX1pRJxD7yPrTo2INcC2ITPb5pn0XhgvmXb5sPt7H9bPfhpwE/DPwJ8Nk0nvoG2TL+KtwPX0NiNcsIi4le2L4SfAK4fJnIwGyMzv9H9wvg84okBuDTHH6dnOjwM7p66x7hyo1jt2TrvV6h07x86xc+yc2dg5dY117/j+CqjQO51a4ImIZZm5FXgcsH7KdcN+nS+g99Fl38rMjRGxP/DyYQIjIoBnA/fPzDf0M/fNzIuHnJWIeAhwbP/s+Zl5xbCZwOH0yucYei/E84F3Fsh9H73SOSEzS+xzCb39OZO7XhQJPLn3kC/4IwL3oPd83W/q8zXsoDnlwHWZ+WvgGf3sxSjnOD3b+XFg59yVbeeU7xyo0Dt2TuvV6p3inQP1esfOsXMqsXNmsnPuyrZ3fH9VQ/He6dKnaL0a+B3gRmB/YG1mZkQcBJydmY8eIvsd9FaVj8/MwyJiL+CLmfmwxZTZz30Z8EJg8uP1nkpvtf30IXM/AtwCfLB/0UnAHpn5jGFya4iID9HbZ/hT9Erod4GLge8AZObrF5BZ9PmKiFdm5l9HxOnMfvT4UxaSW1NETAC30XtMVwMbJ68CVmXm8lHNNgp2zp25dk6FzunnFnvO7JxuqNU7FfuhRpfZOXZOFXbOTHbOnbn2ju+vqqjRO51Z4AGIiKOBe9H7xritf9nBwG45xMeiRcSGzFwbUz4qMCKuyMyHLKbMfsaVwCOnfP27Ahdl5lCbpUXEtzLz8J1dNo+8j2TmMyLiKrZ/AQa9/SYXPG9EnAc8KTNv7Z9fA3wuMx8zRGbR5ysibsrMu0XEy4Gbp1+fmWcvdFY1x86xc/rZxTunn1PsObNzuqNG71TshxpdZufYOWrQuHdOP8Pe8f1Va3RmF63Jb5Dpl2fmdTu7zQC2RMRS+i+UiNiH4fcTrZEJvRfwxJTzE1Bkv+ENEXF0Zv4HQEQ8ArhkiLyX9f9+8tCTzXRP+geE69vcv2wYpZ+vn0bEfsDzgOMo8xxVNcjrZ4jXWOvYOXeyc+p0DpR9zuycDqjYO7X6oUaunWPnVGHnzGTn3Mne8f1VFTV6pzMLPMBh/dXVuQS9/fwW4jR6H+F3j4h4I/AH9PYbHUaNTIAzga9HxCf7538feG+B3KOACyPiB/3z+wPXTq4Qz3dFODN/3P/7+wVmm+79wMXTHoOzhsws/Xy9A/gyvYOoXTrl8qBXckMdXG02EbFvZv5kiIiar7E2snN67Jw6nQNln7PGOweG7h07Z6Zaj0mtfqiRa+fYOXOyc4qzc3rsHd9fzWoxvr/qzC5aEXHAADebyMwbFph/KL2DiwXw5cy8ZiE5tTP7uWvpHawLegcBu6xA5g4f3/kWScw8avqdV/Xicvf55M2Sv5a7DoR2XqHHoMb3wDsy88XD5gx4X5/LzCcN8e+rvsbaxs7ZLtfOqdA5/dyiz1mTndO/vwX3jp0zU83HpGI/1OgyO8fOmev+7JyC7Jztcu0d31/Ndl+L7v1VZxZ4JEmSJEmSxtWSUQ8gSZIkSZKk4XR2gSci1o97bptmrZXbpllr5bZp1rbz+fMxqJXbpllr5do5M/n8tSu3TbPWynXWdmvT81crt02z1sp11sWd29kFHqBWKbcpt02z1spt06y1cts0a9v5/PkY1Mpt06y1cu2cmXz+2pXbpllr5Tpru7Xp+auV26ZZa+U66yLO7fICjyRJkiRJ0lho3UGWV8TKXMWuO73dFjaxnJUD5+52+LaBbnfbzZvZda8VA932lh/tNvD9b910G8tW7vzrAljyy40D3W6+j8Gg5p0bMVhu3sHyWDVY5oDft4vmMRhh7mKY9Q5uY3NuGuwbYZFZtmrXXLFm74Fuu/WO21i2arDXce4xMdDtJm7ZyNLddxnotsu/u2mg28H8Xm8x4GsYYHPewYoBc3PbYL27GL6HR53bpllr5Y5L56xYsjpXL10z0G03b7udFUtWD3TbXQ7eMvAMG2/exC577fyxvu1bgz/E8/qemEfnzKvLBk6FzWxixaDzLls6WOY8nq/cOtjPCKjz+xOM/nU86tyx6ZxYmatisN9dtuQmlsdgj0ksGex1AYP/7jDo7w2TBn5tVHpd3O1BmwfOvfUXW1iz9/KBbvuLbw/WIwCbt93BiiUDPLbz6ZyOvo7blnsrN9+YmftMv3xZ8akqW8WuPGLpbxfPPfrDdxTP/OrrHl08E2D1Zy6tksu2wV/Y8xHLB1sQm4/cOvgvqv+3vXsPsvQgywT+vHPJTEJuIAl3DHdQFgMMLOCCUbFEFlxQLDSlFmzpFEFE2YIV1GUtFpSCLVjxghsoCLuwaAki3hAVRSAoEGAgiSEgSEwIt0C45Doz3e/+MR2rGWaY6env+8583b9f1VROnz79fO+ZnnnmnDff6bO24HktPOfiff2ORY9wzE445Ta57xOeNXju3sd/ZfDMO/7YJwfPTJLaPs4/Fcs3HN2yes3W8KDyqPXaHlQefa7OGcOcO+fErafk4bd50uC5D3rj5wbP/MCDhv/3PUlq6wh/h5PU1nFOXN9y+mmDZy5dO/y/EUnSNx/9/wjg6M25c3bWrfKwbT84eO6WU45uUb0Wy9ddP3hmMt7zip/+o38dJfcNj/iuwTOXvvTlwTMZAjPsawAAGupJREFU19/0m6441PVeogUAAAAwcxY8AAAAADNnwQMAAAAwcxY8AAAAADNnwQMAAAAwcxY8AAAAADM32IKnqk6vqqevXD6nqv7sMLd7dVV9x1DHBTYvvQNMSecAU9I5wFoNeQbP6UmefqQbdffPdPc/DXhcYPPSO8CUdA4wJZ0DrMmQC54XJ7lHVe1J8tIkJ1fVm6rqY1X1hqqqJKmqd1bVrqraWlUXVNUlVXVxVT1rwFmAzUHvAFPSOcCUdA6wJtsGzHpukvt399lVdU6Styb5ziRXJ7kwyXcnec+q25+d5E7dff/kwCmIhwuuqt1JdifJzpw04MjAzI3SO6s7Z/vJtx5teGB2Ru+cnVtOHm14YHbG7xzPrWBDGfOHLL+/u6/q7uUke5KcddDnP5Xk7lX1W1X1mCRfO1xQd5/f3bu6e9f27BhvYmDuBumd1Z2zbeetxp0YmLPBO+eELSeOOzEwZ4N3zvby3Ao2kjEXPDevuryUg84W6u5rk3xXkncmeVqSV484C7A56B1gSjoHmJLOAb6lIV+i9fUkpxztjavqtkn2dvebq+ryJK8fcBZgc9A7wJR0DjAlnQOsyWALnu7+UlVdWFWXJLkxyeeP8CV3SvLaqrrlLKLnDTULsDnoHWBKOgeYks4B1mrIM3jS3ece5vpnrLp8zqpPPWjI4wObj94BpqRzgCnpHGAtxvwZPAAAAABMwIIHAAAAYOYseAAAAABmzoIHAAAAYOYseAAAAABmbtB30ZpCnXBCtt35zoPnvv8R1wyeueWcHjwzSbbd8faj5O779jNGyf3U408cPPPuz/vHwTOTJFXj5PY4fxaYwJZk/0nD/7m4y3nXDp557Y+O88YZp37i66Pkbrn0k6Pk5p5nDR65fOnlg2fCoXWyvDR46ocedevBM7/8J+M8HjnjJ4/0TtDHpvfuGyV3392H/32o947zewAHq+3bs/WOdxg8d/+VVw+eufV+9xw8M0mWLvvnUXLf8LjvGSX38xcM/xT+to//8uCZLIYzeAAAAABmzoIHAAAAYOYseAAAAABmzoIHAAAAYOYseAAAAABmzoIHAAAAYOYWsuCpqvcu4rjA5qV3gCnpHGBKOgdIFrTg6e5HLOK4wOald4Ap6RxgSjoHSBZ3Bs91K/+9Q1W9q6r2VNUlVfXIRcwDbHx6B5iSzgGmpHOAJNm24OOfm+Tt3f2iqtqa5KRD3aiqdifZnSQ7t50y4XjABnTE3lndOdtPufXE4wEbzJo6Z+eWkyceD9hg1tY5Wz23go1k0QueDyR5TVVtT/LH3b3nUDfq7vOTnJ8kp+24fU84H7DxHLF3VnfOSbe7i84B1mNNnXPa9jN0DrAea+ucHbfTObCBLPRdtLr7XUkeleQzSS6oqp9e5DzAxqd3gCnpHGBKOgc2t4UueKrq25N8vrtfleTVSR60yHmAjU/vAFPSOcCUdA5sbot+idY5SZ5TVfuSXJfEhhkY2znRO8B0zonOAaZzTnQObFoLWfB098kr/31dktctYgZgc9E7wJR0DjAlnQMkC36JFgAAAADrZ8EDAAAAMHMWPAAAAAAzZ8EDAAAAMHMWPAAAAAAzt+i3SV+z3rs3+6+4avDcLfe/1+CZJ73vU4NnJsnyddePktt3O3OU3DP29OCZW3bsGDwzST73nx80Su7tL9gzeObyDTcMnsk3237t3tzpLVcMntunnjx45q3//tODZybJ5f/zDqPk3utp20fJreXlEUJH+v8hvTROLvO1dVvybbcePHbpNrcaPPM2P3zp4JlJkjO+bZTYfWffY5Tczz585+CZd/3I8N+vJFm+fpzHkKkaPrOHf/yYZKRZh4+cSu/dl/1XXLnoMY7K0qWXL3qENXnT3/3+KLk/ctYjBs+c8R9hDuIMHgAAAICZs+ABAAAAmDkLHgAAAICZs+ABAAAAmDkLHgAAAICZs+ABAAAAmDkLHgAAAICZG3zBU1WnV9XTVy6fU1V/NvQxAFbTO8CUdA4wJZ0DHK0xzuA5PcnTR8gFOBy9A0xJ5wBT0jnAUdk2QuaLk9yjqvYk2Zfk+qp6U5L7J/lgkp/s7q6qByd5WZKTk1yT5Cnd/dkR5gE2Pr0DTEnnAFPSOcBRGeMMnucm+WR3n53kOUkemOQXk3xHkrsn+e6q2p7kt5I8qbsfnOQ1SV50uMCq2l1VF1XVRfty8wgjAzM3aO+s7py9yzdOcgeAWRmvc5ZumOQOALMyWud4bgUbyxhn8Bzs/d19VZKsbJ3PSvKVHNg4/3VVJcnWJIfdLnf3+UnOT5JT6zY98rzA/K2rd1Z3zmkn3E7nAEcyXOfsvIPOAY5ksM7x3Ao2likWPKvXwksrx6wkl3b3wyc4PrD56B1gSjoHmJLOAQ5pjJdofT3JKUe4zeVJzqiqhydJVW2vqu8cYRZgc9A7wJR0DjAlnQMclcHP4OnuL1XVhVV1SZIbk3z+ELfZW1VPSvKKqjptZY7/leTSoecBNj69A0xJ5wBT0jnA0RrlJVrdfe5hrn/Gqst7kjxqjOMDm4/eAaakc4Ap6RzgaIzxEi0AAAAAJmTBAwAAADBzFjwAAAAAM2fBAwAAADBzFjwAAAAAMzfKu2iNqpLaUoPHLl98+eCZqXH2Z2Pc/yQ57devHCX3a6+61+CZy3v3DZ6ZJGe+8h9GyV3uHiWX8fX+/Vn64jXD5169f/DMsdz1dXceJXffruG7IUk++ZPDd++9f3bwSDik3rs3y1dcNXhufWpp8MxeHj4zSZa++KVRcj/1knG67NQPDp+5fONNw4eOaU6Pc+Y0K7N20pYTRsnt/fN5DMn0nMEDAAAAMHMWPAAAAAAzZ8EDAAAAMHMWPAAAAAAzZ8EDAAAAMHMWPAAAAAAzZ8EDAAAAMHODLXiq6plVdVlVXVtVz13D151VVecONQewOegcYGp6B5iSzgHWatuAWU9P8ujuvupQn6yqbd29/xCfOivJuUn+34CzABufzgGmpneAKekcYE0GWfBU1e8luXuSt1XVa5Lco7ufUVUXJLkpyQOTXFhVb03ymytf1kkeleTFSe5XVXuSvK67Xz7ETMDGpXOAqekdYEo6BzgWgyx4uvtpVfWYJN+b5HEHffrOSR7R3UtV9adJfq67L6yqk3OgnJ6b5NndffDX/Zuq2p1kd5LszElDjAzMmM4BpjZm7+gc4GA6BzgWU/yQ5T/s7qWVyxcmeVlVPTPJ6Yc5pfCbdPf53b2ru3dtrx2jDQpsCAN3zs7RBgU2jHX1js4B1mi4zonnVrCRTLHguf6WC9394iQ/k+TEHDil8L4THB/YXHQOMDW9A0xJ5wCHNOQPWT6iqrpHd1+c5OKqekiS+ya5MskpU84BbA46B5ia3gGmpHOA1aY4g2e1X6yqS6rqo0n2JXlbko8mWaqqj1TVsyaeB9jYdA4wNb0DTEnnAP9msDN4uvuslYsXrPxKdz/loNv8/GG+/PuGmgPYHHQOMDW9A0xJ5wBrNfUZPAAAAAAMzIIHAAAAYOYseAAAAABmzoIHAAAAYOYseAAAAABmbrB30ZpMJ720NEJuD5+Z5REyk94/xqzJ9Y+5cZTc933ilYNnPuaNDx08M0lq+45RcpdvuGGUXKbQyRidszxC5khO+Ns9ix5hTfb833cPnvnkLY8cPDNJepx/Jpiz7vS+/cPnzqhzxpr13rsvHSX3lR9/x+CZ571ynDdA6jn9OWAaldS24Z8S9v4Remxm7vX680bJ7ZcOn3nPX/7g8KFJet/eUXI5PGfwAAAAAMycBQ8AAADAzFnwAAAAAMycBQ8AAADAzFnwAAAAAMycBQ8AAADAzB3TgqeqzqqqS9Z78Kr6dFXddr05wMand4Ap6RxgSjoHGIIzeAAAAABmbj0Lnm1V9Yaquqyq3lRVJ1XV91fVh6vq4qp6TVXtSJLDXX+Lqjqxqt5WVT+7rnsDbHR6B5iSzgGmpHOAdVnPguc+SX63u++X5GtJ/kuSC5I8ubv/XZJtSc6rqp2Hun5VzslJ/jTJG7v7VYc6UFXtrqqLquqifbl5HSMDMzdJ73xD57TOgU1s+s7xOAc2M49zgHVZz4Lnyu6+cOXy65N8f5J/6e6Pr1z3uiSPyoGiOtT1t3hrktd29/853IG6+/zu3tXdu7Znx+FuBmx8k/TON3RO6RzYxKbvHI9zYDPzOAdYl/UsePqgj79yjDkXJnlMVdU6ZgE2B70DTEnnAFPSOcC6rGfBc9eqevjK5XOTXJTkrKq658p1P5Xk75Ncfpjrb/H8JNcm+Z11zAJsDnoHmJLOAaakc4B1Wc+C5/IkP1dVlyW5dZKXJ3lqkj+sqouTLCf5ve6+6VDXH5T1C0lOrKqXrGMeYOPTO8CUdA4wJZ0DrMu2Y/mi7v50kvse4lPvSPLAQ9z+cNefterDpx7LLMDmoHeAKekcYEo6BxjCes7gAQAAAOA4YMEDAAAAMHMWPAAAAAAzZ8EDAAAAMHMWPAAAAAAzd0zvorVw3Yue4OjMZc4Vy9dfP0ruD97x7BFS946Qmfzy5e8fJfd/PHX4NzHY8vcfHjyTb1ZbtmbLybcaPHfpK18dPHMstW2cfyq2nH7aKLnnPuSJg2d+6oV3GzwzSe7+pq+Pkrv1i8P/+Vq+5suDZyZJjfD3q66Z58ObJElVavvw8/fNS4Nnzs7SOL8H5z3pvMEzP/GS4f9eJMm9n3/pKLlj/N7W3e4yeGaSZO++wSPrincPnjkVj3PGc+/f/cw4wfv2Dx55w/c9YPDMJFnePs75JCd+7obBM+um4X9fk6Su+uwoubn20Fc7gwcAAABg5ix4AAAAAGbOggcAAABg5ix4AAAAAGbOggcAAABg5ix4AAAAAGbOggcAAABg5o6rBU9VvXfRMwCbh84BpqRzgKnpHdhcjqsFT3c/YtEzAJuHzgGmpHOAqekd2FyOqwVPVV236BmAzUPnAFPSOcDU9A5sLtsWPcDRqKrdSXYnyc6ctOBpgI3uGzpny60WPA2w0XmcA0zJ4xzYuI6rM3gOp7vP7+5d3b1re3Ysehxgg1vdOSfUiYseB9jgvuFxTu1c9DjABudxDmxcs1jwAAAAAHB4FjwAAAAAM2fBAwAAADBzx9WCp7tPXvQMwOahc4Ap6RxganoHNpfjasEDAAAAwNpZ8AAAAADMnAUPAAAAwMxZ8AAAAADMnAUPAAAAwMxtW/QAx6Rq+Mzu4TNJktT2EwbP3HrmbQfPTJLn/9xDRsnd+d+vHj70h3YMn5lky47hc+u6+e6Se2kpS1/56qLHWKjlm24aJ/dz4+SO4W7P+/w4wVu3jhL75/960eCZP3jHswfPTJJcf/3gkd37B8+cTHf65psXPcWG1PtH+nPxgYsHj7znBwaPTJIsjxObt1+9Z/DM0TpnBN17Fz3CMfM4Zzz7r7hynOARnreecPVnB89Mkq33vNsouXd6/fCPy67+8XGeXy7dOO3j3fk+6wIAAAAgiQUPAAAAwOxZ8AAAAADMnAUPAAAAwMxZ8AAAAADMnAUPAAAAwMxZ8AAAAADM3LYpDlJVv5bkuiSnJnlXd//NQZ8/J8mzu/txU8wDbHx6B5iSzgGmpHOAQ5lkwXOL7n7+lMcD0DvAlHQOMCWdA6w22ku0qupXqurjVfWeJPdZue6CqnrSyuXHVNXHqupDSX5krDmAzUPvAFPSOcCUdA5wJKMseKrqwUl+PMnZSR6b5CEHfX5nklcleXySBye5/RHydlfVRVV10b7cPMbIwMwN2Ts6BzgSnQNMSecAR2OsM3gemeQt3X1Dd38tyZ8c9Pn7JvmX7v5Ed3eS13+rsO4+v7t3dfeu7dkx0sjAzA3WOzoHOAo6B5iSzgGOyLtoAQAAAMzcWAuedyV5QlWdWFWn5MCpgqt9LMlZVXWPlY9/YqQ5gM1D7wBT0jnAlHQOcESjvItWd3+oqv4gyUeSfCHJBw76/E1VtTvJn1fVDUneneSUMWYBNge9A0xJ5wBT0jnA0RjtbdK7+0VJXvQtPv+XOfBaUYBB6B1gSjoHmJLOAY7Ez+ABAAAAmDkLHgAAAICZs+ABAAAAmDkLHgAAAICZs+ABAAAAmLnR3kVrVN2LnoA16H17B89c+sI1g2cmya0uGWfnefmH7zJ45qlPvevgmUly5gevGzyzL9kxeCZMqU44YZzcrVtHyX3wr503eOb+Z9XgmUlyypVLg2cu/80/Dp7JBlDj/BkeQ23bPkruGI/JkuQ/PuKHB8/c+9fjfL+2vfA2w4de9N7hM5m9bbc7c5Tc/Z//wvChIz2/Xv70VaPkfuYHdg6e+e/f/YnBM5PkfY++0yi5uenQVzuDBwAAAGDmLHgAAAAAZs6CBwAAAGDmLHgAAAAAZs6CBwAAAGDmLHgAAAAAZs6CBwAAAGDmLHgAAAAAZs6CBwAAAGDmti16gKNRVbuT7E6SnTlpwdMAG53OAaakc4Ap6RzYuGZxBk93n9/du7p71/bsWPQ4wAanc4Ap6RxgSjoHNq5ZLHgAAAAAODwLHgAAAICZO24WPFX1F1V1x0XPAWweegeYks4BpqRzYPM5bn7Icnc/dtEzAJuL3gGmpHOAKekc2HyOmzN4AAAAADg2FjwAAAAAM2fBAwAAADBzFjwAAAAAM2fBAwAAADBzx827aMFa9L69o+Tuv/KqUXLv87LlwTP75psHz0yS8z/01sEzH/fYawbPhCn13nE6p7tHyT3jNR8YPHP/f3jA4JlJcs4r3jt45hWXfn3wTDaAkf6+jWGsxzlj2X/FlYNnnvSUOwyemSQveM//Hjzzpx/vcQ7fbP/nPr/oERZurC5b2r9v8Mx3/8LDBs9Mkof+1UWj5L797ENf7wweAAAAgJmz4AEAAACYOQseAAAAgJmz4AEAAACYOQseAAAAgJmz4AEAAACYOQseAAAAgJmz4AEAAACYuSMueKrqrKq6sar2rHy8VFV7Vv167sr176yqi1Z93a6qeufK5XOq6qtV9eGquryq3lVVj1t122dV1b9W1W8Pfg+BWdE5wNT0DjAlnQOMZdtR3u6T3X32yuUbV10+2JlV9UPd/bZDfO7d3f24JKmqs5P8cVXd2N3v6O6XV9W1SXatbXxgg9I5wNT0DjAlnQMMbuiXaL00ya8c6UbdvSfJC5I842hCq2p3VV1UVRfty83rHBHYQHQOMLXBe0fnAN+CzgGO2rEseE486BTCJ6/63D8k2VtV33sUOR9Kct+jOWB3n9/du7p71/bsOIaRgRnTOcDUJu0dnQObns4BBnG0L9Fa7VudQpgkL0zyq0l+6Qg5dQzHBjYfnQNMTe8AU9I5wCAGfxet7v7bJCcmedgRbvrAJJcNfXxgc9E5wNT0DjAlnQMcrbHeJv2FSf7r4T5ZVQ9I8t+S/M5Ixwc2F50DTE3vAFPSOcARHctLtE685S39Vvxldz939Q26+y+q6osHfd0jq+rDSU5K8oUkz+zudxzD8YHNRecAU9M7wJR0DjCINS94unvrYa4/56CPH7zq8juTnLbWYwHoHGBqegeYks4BhnI0L9FaSnLaQVvlQVXVs5I8L8nXxjoGMBs6B5ia3gGmpHOAURzxDJ7uvjLJXcYcortfnuTlYx4DmAedA0xN7wBT0jnAWMb6IcsAAAAATMSCBwAAAGDmqrsXPcOarPz0+CuO4qa3TXLNCCPMKXdOs46VO6dZx8o9Hmb99u4+Y4QZRreGzkk27vdv0blzmnWs3DnNOlauzvlmG/X7t1Fz5zTrWLkbdVadsz5zyp3TrGPlmvX4yD1k78xuwXO0quqi7t61mXPnNOtYuXOadazcOc06d75/fg/Gyp3TrGPl6pxv5vs3r9w5zTpWrlnnbU7fv7Fy5zTrWLlmPb5zvUQLAAAAYOYseAAAAABmbiMveM6XO6tZx8qd06xj5c5p1rnz/fN7MFbunGYdK1fnfDPfv3nlzmnWsXLNOm9z+v6NlTunWcfKNetxnLthfwYP06qq67r75FUfPyXJru5+xgDZ70zy7O6+6KDrL0jyPUm+unLVU7p7z3qPBxz/FtQ5leSFSX4syVKSV3b3K9Z7POD4t6DOeXeSU1Y+PDPJ+7v7Ces9HjAPC+qd70/y0hw4EeS6HHh+9c/rPR7T2bboAWCdntPdb1r0EMCm8JQkd0ly3+5erqozFzwPsIF19yNvuVxVb07y1gWOA2wOr0zyn7r7sqp6epJfzYHHP8zERn6JFseJqjqjqt5cVR9Y+fXdK9c/tKr+oao+XFXvrar7rFx/YlX9flVdVlVvSXLiQu8AMCsjds55SV7Q3ctJ0t1fmOQOAce1sR/nVNWpSb4vyR+PfmeAWRixdzrJqSuXT0ty9eh3hkE5g4ehnFhVq18edZskf7Jy+TeTvLy731NVd03y9iT3S/KxJI/s7v1V9egkv57kR3PgSdQN3X2/qnpAkg99i+O+qKqen+QdSZ7b3TcPe7eA49QiOuceSZ5cVU9M8sUkz+zuTwx+z4Dj0aIe5yTJE5K8o7u/NuD9AY5/i+idn0nyF1V1Y5KvJXnY4PeKUVnwMJQbu/vsWz645TWiKx8+Osl3HPjxFUmSU6vq5BzYCr+uqu6VA9vi7Suff1SSVyRJd3+0qj56mGM+L8nnkpyQAz+Q6peSvGCoOwQc1xbROTuS3NTdu6rqR5K8JskjD3NbYGNZROfc4ieSvHqIOwHMyiJ651lJHtvd76uq5yR5WQ4sfZgJCx6msCXJw7r7ptVXVtVvJ/m77n5iVZ2V5J1rCe3uz65cvLmqXpvk2esfFdgARumcJFcl+aOVy29J8tr1jQlsEGN1TqrqtkkemuSJ6x8T2EAG752qOiPJd3X3+1au+oMkfznItEzGz+BhCn+V5Odv+aCqbtlEn5bkMyuXn7Lq9u9Kcu7Kbe+f5AGHCq2qO6z8t3Lg9OVLhhwamK1ROicHfv7F965c/p4kHx9mXGDmxuqcJHlSkj87+EkcsOmN0TvXJjmtqu698vEPJLlsuJGZggUPU3hmkl1V9dGq+qckT1u5/iVJfqOqPpxvPJvslUlOrqrLcuAlVx88TO4bquriJBcnuW0OvH0xwFid8+IkP7rSO78RpywDB4zVOUny40neOMLMwLwN3jvdvT/JzyZ5c1V9JMlPJXnOiPeBEVR3L3oGAAAAANbBGTwAAAAAM2fBAwAAADBzFjwAAAAAM2fBAwAAADBzFjwAAAAAM2fBAwAAADBzFjwAAAAAM/f/AT3CKxGTS6PzAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1152x576 with 8 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PIlNeqb5Cbx5"
      },
      "source": [
        "The model is robust for unfamilier words: in the following example, neither \"triceratops\" nor \"encyclopedia\" are in the input dataset, but the model almost learns how to transliterate them."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 565
        },
        "id": "o2iOqkr6Cs0g",
        "outputId": "77113fd4-4766-4d64-f29d-3369ccaa4ff5"
      },
      "source": [
        "sentence = \"Eu li sobre triceratops na enciclopédia.\"\n",
        "ground_truth = \"I read about triceratops in the encyclopedia.\"\n",
        "\n",
        "translated_text, translated_tokens, attention_weights = translator(\n",
        "    tf.constant(sentence))\n",
        "print_translation(sentence, translated_text, ground_truth)\n",
        "\n",
        "plot_attention_weights(sentence, translated_tokens,\n",
        "                       attention_weights['decoder_layer4_block2'][0])"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input:         : Eu li sobre triceratops na enciclopédia.\n",
            "Prediction     : i read about triumpering on enclopedia .\n",
            "Ground truth   : I read about triceratops in the encyclopedia.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAIgCAYAAAD+/97jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZxkZX3v8c+vt9kXBgZQkH1gRMARRtm3oCFuBOOWqFdcbiYxMcZ4xWhijNEkkwhZFDVx9KoYb5JxS4waF6IMiA7CADPDOoogsg4zMMAwS0931+/+cU5r0VPVXb/uOk+d0/V9v171mupzfvXUU6ervn3qmXOeY+6OiIiIiIiIiIhUV0+nOyAiIiIiIiIiIlOjAR4RERERERERkYrTAI+IiIiIiIiISMVpgEdEREREREREpOI0wCMiIiIiIiIiUnEa4BERERERERERqTgN8IiIiIiIiIiIVJwGeEREREREREREKk4DPCIiIiIiIiIiFdfX6Q6kZGYfaaHsCXd/b+GdEZFpT5kjIikpc0QkJWWOSPmYu3e6D8mY2T3A+yYoe7e7PzNFf0RkelPmiEhKyhwRSUmZI1I+XXUED/AP7n75eAVmtk+qzojItKfMEZGUlDkikpIyR6Rkum0OnuGJCtz9H1N0RES6gjJHRFJS5ohISsockZLptgGeN3W6AyLSVZQ5IpKSMkdEUlLmiJRMtw3wSBNm1mtmf9TpfohId1DmiEhKyhwRSUmZI53SbZMsDwM7G60C3N3nJ+5SqZjZde7+vE73Q9ovP/95CTBzdJm7X925HnUHZc74lDnTlzKnM5Q541PmTF/KnM5Q5oxPmTO9lTV3um2A5yZ3f06n+1FWZvYPQD+wGtgxutzdb+xYpzrMzN7l7h8ys8uAvT4s7v62DnQrxMz+N/CHwMHAeuAUYK27/0pHO9YFlDnjU+bsTZkjU6HMGZ8yZ2/KHJkKZc74lDl7mw6ZA+XOnW67ipaMb1n+7wfqljnQ8TdqB92e/7uuo72Ymj8Engtc6+7nmtlS4K873CcRUOY0oswRKY4yZ2/KHJHiKHP2Nh0yB0qcO902wPPFRgvN7FeBi939BYn7Uyrufm6R7ZvZicAZZMH2gyqMXrv71/J/x70EZEpmZsBrgSPc/QNmdghwoLtf1+Qhu919t5lhZjPc/Q4zO6ZNfflIC2VPuPt72/F8FaTMGYcyZ2/KnAn7oswZnzJnHEVnDlQvd8qYORDOHWVO5yhzxqHM2ds0yRwoKHfakTndNsnytWb2YzN70sw+b2bHm9k6YCXwT53uXKeZ2QFm9n/N7Jv5z8ea2Zvb1Pb7gMuBfYH9gM+YWWX+GJrZFWa2sO7nfczs2x3qzseBU4Hfyn/eDnxsnPr78r7/J3CFmX0VuKdNffl14IYJbi9v03NVkTJnHMqc5pQ5TSlzxqfMGUeRmZO3V9ncKVnmQCx3lDmdo8wZhzKnuYpnDhSXO1PPHHfvmhtwE3AOMAO4EHgSeGsH+3M08F3glvznE4D3drA/3wReBWzIf+4Dbm5S+4etLKtbtwmYWffzLGBTp98TgW2zvtH7qUN9uXHs84/+zlp47NnABcBAm/ry9nbUTNebMmfC/rScOfn6lnNHmdPWvihzKnJT5kzYn8IyJ19X2dwpU+bkzz2p3FHmJP89lSpz8j6VJneUOeNum2mROXld23KnHZnTbUfw4O5r3H3Q3f8TuN/dP9rB7nwSeA8wlPdtI/CbHezPfu7+BaCW92cYGGlSe1GDZW8Yp+0HqJthnOwPwf2T6GOnjOSH6gFgZofSYGKwqTCzuWY2t4XSITPrHX1+M1tM/jsb0978/N9FozfgZuAaoJXnmZC7/2M7aqYzZc64IpkDsdxR5rSgxdxR5lSIMmdcRWYOVDt3ypQ50ELuKHPKoWSZA+XKHWVOc5XLnHx5obnTjszptjl4FpjZb9T93Ff/s7t/JXF/Zrv7ddkpf78wnLgP9XaY2b788o19CvB4fYGZ/RbwGuBwM/uvulXzgEfHaftx4FYzuyJv/wXAdaPnGXr5Z0z/U+AaM7uK7NKPZwIr2tGwmR0PfA5YlP1oW4CL3P2WJg/5CPAfwAFm9lfAK4BGh2P+K/ASskP5PO/3KAeOaFP/zwX+ABg97/R24KPuvqYd7VecMmd8E2ZOvnwyuaPMGUcwd5Q51aHMGV+RmQPVzp0yZQ60ljvKnM4rW+ZAuXJHmdNcFTMHEuTOVDOn2y6T/plxVru7vylZZ4D8fMy3Al909xPN7BXAm939hSn7UdefE4HLgOOAW4DFwCvyke/RmkOBw8nOrX133cO3AxvzkelGbTcakf4FL9lEW42Y2X5kl8CDbMb0rW1q94fAn7r7lfnP5wB/7e6njfOYpcB5+Y/fc/fbm9UWycxeDHyU7OoAN5KF3IlkgfhWd//vTvSrLJQ5E/ZnwszJ68K5o8yZsO1Q7ihzqkGZM2F/Csuc/HGVzp0yZU5e0/HcUeaMr2yZA+XKHWXO+JQ5Dfsw5czpqgGe8ZjZAe6+OfFzHgGsAk4DtgF3A6919ylN0GRmX3D3V5nZzTz1UDcjC9sTxnlsH9looZGdwzk0lb6MaXuA7LxY2t12CmZ2EHAodUe+ufvVbWh3g7s/e6JlY9ZPOGN+XtNUo8dEmdkasnODN4xZfgJwmbufPdXnmK6UOb94rDKniaIyJ287lDvKnOpT5vzisYVlTt5+ZXOnTJmTrx83d5Q55daJzMmft1S5o8xprmqZU1fT1FRzpx2Z022naD2FZTNfv5zskLhnAk9P+Ny9wO+5+/PNbA7Q4+7b29T8H+b/viTYp98H/p+735r/vI+Z/Za7f7xB7SlkI9LPBAaAXmCHu89v0vY5ZLO8/4ws4J5hZhc1+xCb2ZHAfe4+mD/2BOBz7v5Y5DW1i5n9LfBq4FZ+eT6mA3v1v247Ppb/vA/QcDvm7jKzPwP+Jf/5dcBd4/TlfcArgS+TbcvPmNkX3f0vx5T+Xf7vTGA5sCGvPwFYRzZT/FQdODaAIDvf2cwOaEP704oyZ68+tZw5+fqWc6ebMievLyx3lDnVpczZq0+FZU5efw4VzZ0yZU7eXiu5o8wpmU5mTv78pcodZU5zFc0cKD53pp45XoJZtFPeyGYX/03gv4B7gcfIZn/v6UBfri2w7V7gyuBjWp7NnOwNfBTZ7Pm9wBuBleO0fQNwTN3PRwM3jNcXsgHIo4AfA5cA/93B980mYEa7t2O+bh+y8z5vzG8fBvaZoC8tz5gPfAU4vu7n44AvtWm7jPc7bLqum27KnHEfE/2stJw73ZQ5k9yWLeeOMqdaN2XOuI8pLHPy+srmTpkyp64/LeWOMqeztzJlTt6f0uSOMmfcbVPZzMnXF5I77cicrjqCx8z+lWwCp++QjY5+D7jTOzdJ2k2WTaT1RWDH6EJvw4Rk7j5iZjUzW+Due03m1USvmZnn76B8FHxgnOe408x63X2EbJTzJrJZ6xvpd/dNdY/9sZn1j9OXmrsPm9nLyA5Huyxvv1PuAvqBwRZqo9txGxCZBG10xvzd+c8TzZh/jLvfXPd8t5jZMwPPN54j7amTwY0y2jS5YZUpcyYU+qzkz9Nq7nRT5kCxuaPMqQhlzoSKzByodu6UKXMgljvKnA4pYeZAuXJHmdNclTMHisudKWdOVw3wAMeSnYt5O3B7/iHt5CREM4FHgF+pW+ZkI4Lt8CRws2Uzq9cHXLM3+7eA1Wb2ifzn38mXNbLTsnM+15vZh4AHgZ5x+rLOzD4FfD7/+bVko9TNDFk2o/xFwEvzZeMFVtF2kr3W71IXRE22ZUvb0cy+xjiXA3T3C5qsis6Yv7HBtt9Ie/z6OOsubdNzVJkyh7ZlDsRyp5syB4rNHWVOdShz6FjmQLVzp0yZA7HcUeZ0TtkyB8qVO8qc5qqcOVBc7kw5c7pukmXLZsf+LbJz/raSTXp1nHdgErCiWZOZ1b3JjOpm1kN2ebrn54uuAD6VjyCPrT0U2Ew2cvpHwALg4+5+Z5O2ZwC/TzZxFcD38/qGo7Zmdizwu8Bad/83MzsceJW7/22j+qJFtmWr29HMxp0ky92vivSlWZ/MbCbwFuCsfNHVwD+5++6xj5X2U+a0J3Py+pZzp5syJ68vLHeUOdWizOlM5uT1lc2dMmXOeP1p1C9lTmd1U+ZAMd8J6uqVORXInLy+tLnTVQM8ZnaKu19b9/NJZIH0KrIJp5peNq2g/hxBdj7gKWQjhWuBt7v73Sn7ITJZZnYlzUfJ3d3Pa7KuKyhzRNpLmTM+ZY5Ieylzxle2zMn7oNyRympH5ox3yNd09JRZtt39Bnd/J9nl2d7dgf78K/AF4Glks8x/Efj3djVuZtfV3X9lO+uLbHsy9UWr8rYpeFu+E7h4zO3LwJFAwxn/u4wyp0P1JfuchGlbNqXMGZ8yZxrXF6lsr7XIjApS5oyvbJkDJcqdMn1OJlNfpLK91hJty6lnjndg1uxO3YAbO92HMf3Z2GDZhja0+0PgE2SXzFtKNgt709ceqS+y7cnUJ/gdVXbbpN6WwNnA/wDXAC/s1O+sTDdlTvr6sn9OtC3buq2UOXtvE2XONKwv+HdUqtdaZEa1YVspc/beJqXKnLxPHc+dMn1OJlNf8O+nVK+1zNtyspnTbadoPUZ2flxD3nzSpXb3Y1F+94/JJib7d7JDsV5Ndvm2ZjOlt9q+AccD3yA7P3FJ/vM/A1e5+zcnW19k25OpL1qVt02qbWlm5wPvJZsg7a/c/cp2tDsdKHPSv5fL+jlplbblxJQ5zSlzqvH3tky5U7bXWmRGTZYyp7myZE7el9LkTpk+J5OpL1LZXmsZt+VUM6fbrqK1Bfi7TncCuIEscCz/+Xfq1jnNL4XXqk+The0T7v4mADPbAHyT7FKGY994kfoi255MfdGqvG0K35Zmdj2wGLiE7BxnzOzE0fXufuNUn6PilDnp38ul+5wEaVuOQ5kzIWVONf7elil3yvZai8yoMGXOhMqSOVCu3CnT52Qy9UUq22st1bZsR+Z02wDPk95k5uxGzOwzNJjkaPSXOVnufvhUHt+ClWRvsAPN7Adko38HAIvIDimbSn2RbU+mHgDLZjJ/M/AssssjAlP/XQX7U7ZtM6ltGbSD7HKRr8hv9ZynXqKyGylz0r+Xuylziq5X5lRPKHOgmNzpssxJUQ8Uljtle61FZtRkKHPGV4rMyR9fptwp0+dkMvWAMqdN9VFTz5xWz+WaDjfgK8H6l9fdXgt8CfhIG/vTD7wtb/dLwFuB/ja2f1P+72xgE9mkTV9tR32RbU+y/ovAB4GfAhcB3wE+3IltWcJtE6rXrX03ZU7n3svdlDlV35a6te8WzZz8MYXlTjdlTqL6wnKnhK+1sIzSrX23smVO3n5pcqdMn5NJ1itz2lSf8tbxDiR9sfBc4MC6n18PfBX4CLCohcf3AD9sY38+BVxONhL3K8BngE+1sf0z6u5P+IaL1BfZ9iTrRz9kG/N/+4FrO7EtS7htQvXB7fKuuvuvHLPur9v5XFW8KXM6V99NmVP1bRncLsqc8bfPlDInf0zbcqebMidRfWG5U8LXWlhGBbeLMmf87VOqzMnbK03ulOlzMsl6ZU6b6gPbZcqZ022XSf8EsAfAzM4C/gb4HPA4sKqFxy8B9m9jf57r7he5+/fy2xvJgrJdzqm7/6o21xfZ9mTqh/J/HzOz44AFtPd3FelPpLaM9RG/WXd/7LnNv9bm56oiZU7n6ovuS5kyp+j6ovsSocwZ31QzB9qbO92UOSnqi8ydaF/KVB9tO0KZM76yZQ6UK3citWWsV+a0r75VU86cbhvg6XX3R/P7rwZWufuX3f3PgKPGFpvZdjN7YvQGfA14Vxv7M2JmR9Y93xHAyFQbNbM/NrNTeep5e2vbUV9k25Opr7PKzPYhm3H8v4DbgL9t4XHjqvK2mcK2jLAm9xv93I2UOYnruylziq5X5lRSKHOg8NyZ9pmTor5O23OnbK+1yIyaJGXO+MqWOVCC3CnT52Qy9XWUOVOsn4QpZ063TbLca2Z97j4MnAesqFvXaFssIDs39HB3/4CZHQIc2Mb+vBO40szuyn8+DHhjG9q9A3glcISZfT//eV8zO8bdN02xvsi2J1M/6l/IzuU9jOywTMgmvNqLmRnZ7/WI+t+ru183xf6UbdtMdltGeJP7jX7uRsqc9O/l0mUOhHKnG7dlhDJnfNHMgWJzpxsyJ0X9qCL2dcr2WovMqMlQ5oyvbJkD5cidMn1OJlM/Spkz9fqoqWdOu84Xq8IN+FPgB2Tnht4EWL78KOAHDer/CfgYcHv+8z7A9W3szyuB+cAJwPvILqt2YhvaPZtspvMbgV7gOOAesnNQ9zrHNVJfZNuTqa973LeA1WT/A/B/Rm9Nalv+vVZ520x2WwbfayPAE8B2YDi/P/rzUNGf6bLfUOYkfy+n+pwQyJzI77Ybt2XwvabMGX/7hDIn8t6cZH+mfeakqK97XNv3dcr2WiP1k92OwfeaMmf87VOqzMnb63julOlzMpn6uscpc9q0LQPvsylnTseDIfUNOAV4GTCnbtnRjT74wI35vzfVLdvQoO4L+b83AxvrbjeTT0rVpC+jE1adAVwJvBj4UZPav21lWb78r4Hvkp3/+mGyy9vdNk4/Wq4vsu3J1Nc97pbAe6Cl32vVt81kt6Vu7b0VkTn58nDuRDInr2spd8r0Xi5j5kR+t924LXVr7y2SOfm6wvZ1uiFzUtTXPa7t+zple61FZpRuxdzKlDl5fce/X5XpczKZ+rrHKXPatC1T3jregaQvNn/jtVoD/IhsZG70Dbu4/k1bV/e0/N9DG93Gea7RmclXAq+pX9ZK38cLt3z9BmA52eGSW4BrgK+1o77ItidZvwo4vsX3QUu/12m0bUL1RX6muu1WVObk68K5E8mcZv0fL3fK9F4uU+ZEf7fdti2L/Ex1220y26fV96YypxT1he3rlPC1FpZRRX+muulWtszJ60vz/apMn5NJ1itz2lRf5Gdqr/VT7USVbsAunjoCPPZ2M/DzuvrXkk0odR/wV2TXuH9lG/vzdbLZ5+8CFgIzGDPSCbwl79eOMX29G/j8BO1/qO7+aNjt1476ItueZP1tZLP4b2Li/1EM/14rvm1C9U3aOLDJ8tBnqttuVcycvG5SuVOm93KZMmcyv9tu2pZN2lDmTOI2me1TZO50U+Ykqi9sX6eEr7WwjBrnOffKHWXOhNusVJmTt1+a71dl+pxMsl6Z06b6Jm0Ukjmj50l2BTM7tIWyEXe/r+4xS8kmDTPgu+5+e4N2t9N40iMD3N3nN+nPbLLLnd3s7j8xs6eRjZJ+p65mAdk5jCuBd9c9fLv/ctb6CZnZs919QxH1Rbbdan2z362739OkfsLf61T6M5naMtbXPe4b7v7iBsvDn6luUlTm5HXh3Gklc/K6KedOmd7LZcic/DGTyp3pvi2bPE6ZMwmT3T5F7et0a+YUVZ9qX6cMr3Wy9ZPNnPyxe+WOMmd8Zcuc/HGl/H5Vps9Jq/XKnGLq6x5XSOZ01QCPiIiIiIiIiMh01NPpDoiIiIiIiIiIyNR09QCPma0oqr7ItqteX6a+lK2+TH1JUd9tyrb99d5sT32Z+lK2+jL1pRuVbfuXqb5MfSlbfZn6UrZ6Zc7E9F5rT32Z+lL1+jL1JUV9WyazquoNWFdUfZFtV72+TH0pW32Z+pKivttuZdv+em9q23Tbtum2W9m2f5nqy9SXstWXqS9lq1fmdHZ7dlN9mfpS9foy9SVFfVcfwSMiIiIiIiIiMh10xSTLA/1zfOaMhXstHxraQX//nL2WD+7fuJ2R7Tvonbd3/Yyf7d67bd9Nv83cu5HZDZYBQ8M76O/bu2127GpczyD9zGjc0ZLXt6tt6+1tWL/HdzPQYNv7yEih/WlHfZn60s767Wzb6u6LW26o4gZ6Z/msvr0v7rBnZBcDvbP2Wu5Dww3baZYjw4tnN6wf3rWDvllPzZH+Rweb9nNPbRcDPWP609/XuHZ4JwN9DZ53pNa4fmQnA70N6r1JfaO+ADT7nDfpj+9u/Hqb5khfk9fboD8+3OT3VJHPYSfqlTlp9M+c4wNzFu21fHj3DvpmNti32Lfxe3n48Z30Ldj7c9V3nzWs3zO8g4Ex+y6+a+99olENf1+Nm87qfZB+G1M/zm5rFd6bWOMX3DDvx9lHn46f2yrUK3N+aaBnls/qm7fX8mZ/z2cdtXfu7HxskNkLG2/7nffsvf8zNLyT/gZ/+21P40xr1pfo33PraXxMRNPvHLXG+zr6HHamvkx9aWd9s9xpvGc7zcycsZCTT/jdluvvfGvjLxTNLHnjrS3X1pY9M9S2rd0Yqh9vZ6ASmuz4NNK7YO9Bu/GMbNsW7Y20yf/4l5peOno6mtU3n9Oe/tqW60ce2Bxqf/Orl7dc+7Qv3hlq2w/cN1Tf8/iOUD2De0LlPn9uqH7kJ3eH6nv3a/31jmx+ONR2WCD/gOrnfYG6LXMG5iziuBe+veV6f93WUPv7vKv13cXarZtCbTf7z5pmmv1nTfMHFPw5CX5ura+/5VofiuWldE63ZQ7ArL55nLb41S3XH/uvsX2dm3/72JZre+7bEmp7ZMsjofqeWY3/g76Z2s6doXqs4JNqaoHc7IllcqhtaatmuVP5U7TM7Ied7oOIdA9ljoikptwRkZSUOSLVVfkBHnc/rdN9EJHuocwRkdSUOyKSkjJHpLoqP8BjZk92ug8i0j2UOSKSmnJHRFJS5ohUV+UHeEREREREREREut20nWTZzFYAKwBmDizocG9EZLp7Sub07n1VCRGRdqrPnIHZ+3S4NyLSDZ66rxO7CIKIpDFtj+Bx91Xuvtzdlze6FLqISDvVZ06jS6GLiLRTfeY0vBS6iEibPWVfp8Hlx0Wk86btAI+IiIiIiIiISLfQAI+IiIiIiIiISMVVfoDH3XUCqIgko8wRkdSUOyKSkjJHpLoqP8AjIiIiIiIiItLtpu1VtOrZrkF6bv5py/VL3jwcan/wvGe3XLvn7Y+E2p7/ytgAem379lA9Pb2x+tpIrD7KveXS8GsVScT3DDH88/taf4DFxtpv+pOPt1x7/seeE2qbzQ+Hymux1uMeLLb5keDrLVQg/0Tq9W0fZNH37m65fuQLsX2Rb/58Xcu1v3bI8lDb0fyzvli9D+0J1YcFP7eF90ckleERatsea7l840mx5r99//9rufZFx54da9xjey8+OBiqt4GBWPt7grlQ5P5CcNtEWX9w2ygzw3QEj4iIiIiIiIhIxVV6gMfM3mBmH+10P0Skeyh3RCQlZY6IpKTMEam2jg7wWKbSg0wiUi3KHRFJSZkjIikpc0S6W/IPv5kdZmabzOxzwC3An5nZ9Wa20cz+oq7uP83sBjO71cxW1C1/o5n92MyuA05P3X8RqR7ljoikpMwRkZSUOSIyqlOTLC8BLgLmA68AngcY8F9mdpa7Xw28yd0fNbNZwPVm9mVgAPgL4CTgceBK4KZOvAARqRzljoikpMwRkZSUOSLSsVO07nH3a4FfzW83ATcCS8nCCeBtZrYBuBZ4Rr78ZGCNu29x9z3A6mZPYGYrzGydma3b47sLfCkiUhGF5k595gwRu9qCiExLyTJnT21XwS9FRCog7fcr7euIlFKnjuDZkf9rwEp3/0T9SjM7B3g+cKq77zSzNcDMyBO4+ypgFcCC3v107VkRKTR36jNnvi1S5ohIssxZMLC/MkdE0n6/6tlXuSNSQp2egOvbwJvMbC6AmR1kZvsDC4BtefgsBU7J638EnG1m+5pZP/DKjvRaRKpMuSMiKSlzRCQlZY5IF+vUETwAuPt3zOyZwFozA3gSeB3wLeB3zex2YBPZYYS4+4Nm9n5gLfAYsL4T/RaR6lLuiEhKyhwRSUmZI9Ldkg/wuPvPgOPqfv4w8OEGpS9s8vjPAJ8ppHMiMi0pd0QkJWWOiKSkzBGRUZ0+RUtERERERERERKaoo6dopeK1GrVdgStp1UZC7Q986/qWaz/xiR+E2n5774tC9d3ER2K/J5FkzLCBgZbLfTB2JYoXLjm95dreow4ItU1PcNx/85ZQ+cOvODZUv/DO2Lbp+8EtoXpOWDJxTc5vvC3Wtmv+SUlj94EzuOP/HN5y/VHveDjU/vkHPafl2vve87xQ24f+3ztD9bVHHg3V98yZE6qn4H0LmzWr5dqRxx6LNa7MkZR6e7EF81sv3zMUav7XXvralmsf/s3W+wFw4Bc3heojrxPA+3pD9fQG973u3xwq9+Hhlmt75s8LtT38ULAvQ3tC9WVifbGhk8h2bycdwSMiIiIiIiIiUnHJB3jM7Mk2tbPMzHR4i4iMS5kjIikpc0QkNeWOiIyq8hE8ywAFkIikoswRkZSUOSKSmnJHpOIKHeAxs/80sxvM7FYzW1G3/B/yZd81s8X5smVmdq2ZbTSz/zCzffLla8xseX5/PzP7mZkNAB8AXm1m683s1UW+DhGpBmWOiKSkzBGR1JQ7IjKeoo/geZO7nwQsB95mZvsCc4B17v4s4Crgz/PazwF/7O4nADfXLd+Lu+8B3gesdvdl7r66yBchIpWhzBGRlJQ5IpKackdEmip6gOdtZrYBuBZ4BrAEqAGjgfF54AwzWwAsdPer8uWXA2dN5YnNbIWZrTOzdUPErsIiIpVVjszxwFX7RKTKSpE5I0/umEpTIlItpcidPbVdU2lKRApS2ACPmZ0DPB841d2fDdwEzGxQOtE1HYf5ZT8bPb4hd1/l7svdfXk/M1p9mIhUVKkyx1p+mIhUVJkyp3du8FLgIlJJZcqdgZ5ZrT5MRBIq8gieBcA2d99pZkuBU+qe8xX5/dcA17j748A2MzszX/6/yA4vBPgZcFJ+f/RxANuBeQX1XUSqR5kjIikpc0QkNeWOiIyryAGebwF9ZnY78DdkhxEC7ACeZ2a3AL9CNpkXwEXAJWa2kWwG99HllwJvMbObgP3q2r8SOFaTgIlITpkjIikpc0QkNeWOiIyrr6iG3X0QeGGDVXOb1K/nl6PQ9cvvAE6oW/TefPmjwHOn3lMRmQ6UOSKSkjJHRFJT7ojIRIqeZFlERERERERERApW2BE8pVMb6XQPAHj7SReE6u/44FGh+g8jY/MAACAASURBVCVvuy5U33vMEaF6v+f+UH1t585QfUTP3Ib/WdHUznOeGaqf+bXYthT5BXd8sLir99V2Ba7S9ZO7Qm33HXxQrDMzYxNKD8+2UP2Dp8baP+Ta4P9b3PrTlkvdJ5qzcmpsRuyCAEW+x8Is9nul4G3ZbWY+sItj/vy2lutHCtz+B69cG6p/7FUnh+p3HBjbLzrwR0+G6mt9sQzpv+PeWPuPb2+51np7Q2378HCovvDPbU+s/2XZV5fW+PAwI5sfbv0B0ffbTU+0XLr4xlqoaQ9+h3j01AND9fuufShUf+9LDwjVH/zfoXJqt/+k5VqbFdvv6pkXm7KpJ3hRgOGHNofqwyK5FsxkopncJjqCR0RERERERESk4io7wGNmC83s9zrdDxHpDsocEUlNuSMiKSlzRKqvsgM8wEJAASQiqShzRCQ15Y6IpKTMEam4Kg/w/A1wZH4Zv0s63RkRmfaUOSKSmnJHRFJS5ohUXJUnWX43cJy7L+t0R0SkKyhzRCQ15Y6IpKTMEam4Kg/wjMvMVgArAGYyu8O9EZHpTpkjIik9JXMsdlUSEZHJ0L6OSPlV+RStcbn7Kndf7u7L+4ldelZEJEqZIyIp1WfOQE/ssrYiIpOhfR2R8qvyAM92YF6nOyEiXUOZIyKpKXdEJCVljkjFVXaAx90fAX5gZrdoEjARKZoyR0RSU+6ISErKHJHqq/QcPO7+mk73QUS6hzJHRFJT7ohISsockWqr9ABPSE9v67W1kcK6MbL1kVD90vd7qN6PPyZWH6qG4ROPDtXf+6uzQvVHXHpry7UjTzwRanvm168P1YtMSZGZU2BG+RPbQ/V3/smzQvVH/PkNofqhM44L1Q+eFavv/866UH2RbGAgVO9Dw7En8FqwPvAXIlIrbec1p7Zrd6e7kQm+FxZ8/eZQ/YOXxjLn4P/cFqrfcs7BofpFuw4M1fuNjwaKg58rs1h90Z/bAv9WSedZfx99i1t//w8/+FDsCTzw/gm+9603dhKLBf98Rj9bA4/F6rc9Z99Q/fw77my5duSRQEZNQu3JJ2MPCG5L64/tS0X44GBhbbdTZU/REhERERERERGRjAZ4REREREREREQqrmMDPGa20szONbMLzew9Y9YdY2aXm1mPma0dp42nm9mXiu+tiFSdMkdEUlLmiEhqyh0R6eQRPCcD1wJnA1ePWXdmvux44JZGDzazPnd/wN1fUWgvRWS6UOaISErKHBFJTbkj0uWST7KcX3LvfOBwYC1wJHBePlJ8JXAZcAiwGZgH1MxsnbsvN7M3AL8BzAV6zewi4OvuHptVU0S6hjJHRFJS5ohIasodERmVfIDH3S82sy8ArwfeAaxx99PrSpblhw2eBnwauNTd6y+tdCJwgrs/amaHJeq2iFSUMkdEUlLmiEhqyh0RGdWpU7ROBDYAS4Hb61eY2Wxg0N0dWAJsGvPYK9x9wuu3mdkKM1tnZuuGqMYlzUSkMMocEUkpbeZ4SS6RLiKdlDR39tR2tanbItJOSY/gMbNlwGeBg4GtwOxssa0HTgVWk4XSQjPbCBwGrDOzle6+Om9mRyvP5e6rgFUA822Rt/FliEhFKHNEJKWOZU7PvsockS7VqdxZMLC/ckekhJIO8Lj7erJDBH8InEF2iOCH3P22vOQCM7sYuAt4BHiRu78rZR9FZPpQ5ohISsocEUlNuSMi9ZKfomVmi4Ft7l4DltaFz6izgGvIZnq/KnX/RGR6UeaISErKHBFJTbkjIqM6McnyFuDF+f1TGqx/aX73gw3WfZbsEMTRn38GaIZ3EWlKmSMiKSlzRCQ15Y6IjEo+wNMJ1tdH7377tlw/svnhAnsTU3uypVNif6HngS0F9STTZ/uH6nuOHY49QY/F6iNcpwpLGtbTQ8+c2S3X17ZvL7A3MbXB2ATRB62JfcbNYp/xmT+NZdrtf/T0UP2S/+ltvbg2Emo7qmfB/FC974pNcOnFdl86yGYM0HPEoS3Xj9z+kwJ7E9OzcEGo/ti/vC/4BLGD1XuHYvsK9/9KrP/P+Pl+LdeOPDLhnLdPYf2x3XoP5n1YMO9DtE/XeWbQV82vkrVdsYnp97l+c6z9zbF9lzkPHxCq//lLYu//hV+Z0XJtdD+wcNEcCX6X7Jnd+v76yLY9sb50SKeuoiUiIiIiIiIiIm2iAR4RERERERERkYqb0gCPma00s3PN7EIze8+YdceY2eVm1mNma6fWTRERZY6IpKXMEZGUlDkiMlVTPYLnZOBa4Gzg6jHrzsyXHQ/cMsXnEREBZY6IpKXMEZGUlDkiMiWTGuAxs0vMbCPwXGAt8L+BfzKz95nZmWa2HvgQ8E7gG8D5ZrYuf+wbzOyjdW193czOye8/mbd9q5n9j5k9z8zWmNldZnZB3eO/mi//iZn9+RRev4hUgDJHRFJS5ohISsocEWmXSQ3wuPvFwJvJLqn3XGCju5/g7h9w9++7+zJgE3AscAXwQndf3kLTc4DvufuzgO3AXwIvAF4GfKCu7nnAy4ETgFea2V5tm9kKM1tnZuv21GJXGhGRcqlc5njs6gwiUi6Vy5yRnZN9qSJSAlXIHBibO/p+JVJGUzlF60RgA7AUuL1+hZnNBgbd3YElZIHUij3At/L7NwNXuftQfv+wuror3P0Rd98FfAU4Y2xD7r7K3Ze7+/KBnlmtvyoRKavqZI7NbP1ViUhZVSdzelu/zKuIlFapMwfG5o6+X4mUUV/0AWa2jGx0+WBgKzA7W2zrgVOB1WTBtDA/1PAwYJ2ZrXT31cAwTx1Yqv8mNJQHF0ANGARw95qZ1ffVeaqxP4vINKHMEZGUlDkikpIyR0TaKTzA4+7rgWVm9kOy0d1PAx9y99vykgvM7GLgLuAR4EXu/q66Jn4G/J6Z9QAHkR0SGPUCM1sE7AIuBN40iTZEpAKUOSKSkjJHRFJS5ohIO012kuXFwDZ3rwFL6wJo1FnANWSzvV81Zt0PgLuB24CPADdOogvXAV8GNgJfdvd1k2hDRCpCmSMiKSlzRCQlZY6ItEv4CB4Ad98CvDi/f0qD9S/N736wwToHXtuk3bl199/fbB1wn7tfGO64iFSSMkdEUlLmiEhKyhwRaZdJDfBUjoH1TGU+6fbpnT8/VD+yfXusfusjofqe2bGJGf2Ou0L1/T84KVS/8/SjW66dsSV2pSK78faJi+r48HCoPqynN1RuPdZ67cBAqO3aTl2BZdqy1t83AL5nT6h+4Nux/+Tzk54Vq79nc6j+wB+Gynnw7Se3XPv0y4KvdSi2LYfvuz9UHxZ8L/TMbH2y8GjmjDzxRKhexue7BxnZFPv7XBYjmx8O1Uf/NtuMGaH6fb4ee28+8onDQ/X39C1pufYZlwYzJ5jfhfPYNC7W1/rXEh8ZKbQv0oLhEWqPPd7pXkyKD8VyZOTOu0P11h/7mzh7Tew7ypxjjw/V/+zdJ7Zce9glG0Jt13bsCNUXLfq75WmLWy7tDeZOp/Z1KjfA4+6fJZuITESkcMocEUlJmSMiKSlzRKaXchzWIiIiIiIiIiIikxYa4DGzlWZ2rpldaGbvGbPuGDO73Mx6zGztZDqTzx4vIgIoc0QkPeWOiKSkzBGRdooewXMycC1wNnD1mHVn5suOB26JNGpmfQDuflqwPyIyvSlzRCQ15Y6IpKTMEZG2aWkOHjO7BDgfOBxYCxwJnGdmXwKuBC4DDgE2A/OAmpmtc/flZtYL/A1wDjAD+Ji7f8LMziGbCX4bsBQ42syedPe5+br3A1uB44AbgNe5u5vZi4C/B3aQXRbwCHd/yVQ3hIiUhzJHRFJT7ohISsocESlCSwM87n6xmX0BeD3wDmCNu59eV7IsP2zwNODTwKXufmu+7s3A4+7+XDObAfzAzL6TrzsROM7dG01N/hzgWcADZEFzupmtAz4BnOXud5vZvzXrs5mtAFYAzOyd26xMREqo8pljcyb3wkWkY6qWO0/JHGJXxBSRzqta5oD2dUSqIHKK1onABrLR4Kdcy83MZgOD7u7AEmBT3epfBV5vZuuBHwH75jUA1zUJn9F197l7DVgPHJY/9111j2kaQO6+yt2Xu/vygZ5ZgZcpIiVR3cyx1i8vLSKlUpncqc+cfmKXAheR0qhM5oD2dUSqYMIjeMxsGdml8w4mO6RvdrbY1gOnAqvJgmGhmW0kC4p1ZrbS3VcDBvyBu397TLvnkB0G2Mxg3f2RVvoqItWnzBGR1JQ7IpKSMkdEijLhETzuvt7dlwE/Bo4Fvgec7+7L3H2Xu18AfBJ4C/A24J/zdavzJr4NvMXM+gHM7GizSR/Ttwk4wswOy39+9STbEZGSUuaISGrKHRFJSZkjIkVp6RQtM1sMbMsP51vq7reNKTkLuIZspverxqz7FHAbcKOZ3UJ2juekRovdfRfwe8C3zOwGYDvw+GTaEpHyUuaISGrKHRFJSZkjIkVodZLlLcCL8/unNFj/0vzuBxusqwF/kt/qrclv9bVz83+fss7d31pXdqW7LzUzAz4GrGvlNYhIdShzRCQ15Y6IpKTMEZEiVPG8y982s4uAAeAmshHrcfnQMMMPPlR4x1ph82JX9LLBwYmL6niwvrY7Vo/XQuVP+/gNofrs70pr7vjw8aG2Fz/zuaH6fS5fG6oPq42EyiOb3oeHg52RccQzp1ajtn17cT0KfE6srz/U9F2fe2ao/vDXbAzVs+HHsfpZsUkc53/lxlC9v/LE1mtHYp9Zenpj9cFMCHMPldd27269OPi3RyYUyx0D62k9F4J/ygvltdj7Mtz+nj2h+uimOeoP7gvVv+qam1uu/bfvnh9q2268feKiOmXbVyhbf7pMeF8HdxgaavkJehcvjvUo8H6Ifs7toAND9X7fg6H6nn0WxtqfFZso/5BVwc/6oU9ruXbrq04Itb3//9wbqh95eEuoPvrdNrovNXJb6/ulvQsXxPrSIZUb4HH3fwD+odP9EJHuoMwRkdSUOyKSkjJHZPqIXCZdRERERERERERKSAM8IiIiIiIiIiIVV+oBHjN7h5ndkt/ebmaHmdntZvZJM7vVzL5jZrM63U8RmR6UOSKSmnJHRFJS5ohMb6Ud4DGzk4A3AicDpwC/DewDLAE+5u7PAh4DXt6xTorItKHMEZHUlDsikpIyR2T6K/Mky2cA/+HuOwDM7CvAmcDd7r4+r7kBOKzRg81sBbACYCazC++siFSeMkdEUpt07ihzRGQS2revY3MK76yIxJX2CJ5x1F8rbYQmg1Tuvsrdl7v78n5il54TEamjzBGR1CbMnadkjilzRGRKwvs6A9rXESmlMg/wfB+40Mxmm9kc4GX5MhGRIihzRCQ15Y6IpKTMEZnmSnuKlrvfaGafBa7LF30K2Na5HonIdKbMEZHUlDsikpIyR2T6K+0AD4C7/z3w92MWH1e3/tK0PRKR6UyZIyKpKXdEJCVljsj0VuZTtEREREREREREpAWlPoJnOhq+/4FOd+GpaiOFNu+DgxMX1dcHao9+y42htr993w2h+vMvXxaqF0nGW/+k+NCeUNNHvOGOWFdC1dC7/36xB/TE/h/Cn9wRql/wxdZzJPpa+w49OFT/0K8+PVS//+c3hup75s0N1fvwcMu1e44/NNR275qbQvWR93xX8tjvq1QK3g+Jvnd8TywzRx59LFT/78cf1nLto/8r9pmd8YyTQvVz7t8Vqu/duj1UX9sn1v+eTfe0XjzQH2p7ZOsjoXqZmLtT27279QdEaov245+Gyq0v9pV5+MHNofqembEJq30klpu+/raWa/d/4rBQ24+ffFCofsuyZ4Tqj/zwnaF63x7MqT1DLdeOHH1IqG2uuzlW3yY6gkdEREREREREpOI0wCMiIiIiIiIiUnEa4BERERERERERqbhKDvCY2evM7DozW29mnzCz3k73SUSmN+WOiKSkzBGRlJQ5ItND5QZ4zOyZwKuB0919GTACvLazvRKR6Uy5IyIpKXNEJCVljsj0UcWraJ0HnARcb2YAs4CHxxaZ2QpgBcBMZqfsn4hMPxPmjjJHRNpImSMiKen7lcg0UcUBHgMud/f3jFfk7quAVQDzbZGuryoiUzFh7ihzRKSNlDkikpK+X4lME5U7RQv4LvAKM9sfwMwWmdmhHe6TiExvyh0RSUmZIyIpKXNEponKDfC4+23Ae4HvmNlG4ArgaZ3tlYhMZ8odEUlJmSMiKSlzRKaPKp6ihbuvBlZ3uh8i0j2UOyKSkjJHRFJS5ohMD5U7gkdERERERERERJ6qkkfwSIVkM/G3zgPztdVGQk3/2gWvC9Xf+fdzQ/UHf68Wqp97w89D9SNbHw3VR/jQnsLalmrxwcHYA4Kf8eH7H4i139MbK58Tu6pHbfv2UH3EyH0Phurn/Ebs/1z2/GRpqH7gh7eG6ultfdv3XrUh1nYk60XqFblfMan62L6IB/ZdFn16bajtvgMPCNU/8ul5ofqFfxrL16H5A6H6gZHWt40/tjPUtshU+PBwoe3Xdu2KPaDAv6HDd/0sVD/3/ti+zrtW3hyq//D3fitUH93Xsf7Wh0P8uljfO0VH8IiIiIiIiIiIVFzHBnjMbKWZnWtmF5rZe8asO8bMLjezHjOL/feFiEgTyh0RSUmZIyIpKXNEpJNH8JwMXAucDVw9Zt2Z+bLjgVsS90tEpi/ljoikpMwRkZSUOSJdLvkcPGZ2CXA+cDiwFjgSOM/MvgRcCVwGHAJsBuYBNTNb5+7LzewNwAXA7Pxx/+Hu70r9GkSkWpQ7IpKSMkdEUlLmiMio5AM87n6xmX0BeD3wDmCNu59eV7IsP2zwNODTwKXuXj9b0jLgOcAgsMnMLnP3exN1X0QqSLkjIikpc0QkJWWOiIzq1ClaJwIbgKXA7fUrzGw2MOjuDiwBNo157Hfd/XF33w3cBhza6AnMbIWZrTOzdUMErwojItNRobmjzBGRMZQ5IpKSvl+JSNojeMxsGfBZ4GBgK9mhgGZm64FTgdVkobTQzDYChwHrzGylu6/Om6lPkxGavAZ3XwWsAphvi3Q9VpEulSp3lDkiAsocEUlL369EpF7SAR53X092iOAPgTPIDhH8kLvflpdcYGYXA3cBjwAv0jmgIjIVyh0RSUmZIyIpKXNEpF7yU7TMbDGwzd1rwNK68Bl1FnAN2UzvV6Xun4hMP8odEUlJmSMiKSlzRGRUJyZZ3gK8OL9/SoP1L83vfrDBus+SHYI4+vNLCumkiEwryh0RSUmZIyIpKXNEZFSnJlkWEREREREREZE2SX4ET8eYtV7rJZozLNLvSeiZOzdU77t2hep7n3FQqL62eUvLtTZjRqjtR46dF6pfetkDofo9By8K1dMTG1+13tbrexbvF2p7+N77QvXSgqpmTlDvkiNC9bW7Y1dd7T3owFj7C+aE6tlw+8Q1k+TDQ6H6B24+IFR/9M83h+pr0ffZnlj/RVKwvv5QffRzaAMDsfq+2K50befOQNux1xrdZ9x1xf6h+vmzW+87gAd3Ye3prWdgz/BIqO3hu++JdUZa0yX7OtYfywV6Ym9+i37f649lQ2379khnQm33zJoZqv+rD1wUqp/LnlD97rOPC9UPzettvS9fvSnUtg/F+t4uOoJHRERERERERKTiOjrAY2YrzexcM7vQzN4zZt0xZna5mfWY2dpO9VFEphfljoikpMwRkZSUOSLdrdNH8JwMXAucDVw9Zt2Z+bLjgVsS90tEpi/ljoikpMwRkZSUOSJdrCNz8JjZJcD5wOHAWuBI4Dwz+xJwJXAZcAiwGZgH1MxsnbsvN7Ne4G+BXwNqwCfd/bIOvAwRqRDljoikpMwRkZSUOSICHRrgcfeLzewLwOuBdwBr3P30upJl+WGDpwGfBi5191vzdSuAw4Bl7j5sZsGZbUWkGyl3RCQlZY6IpKTMERHo7ClaJwIbgKXAUy5jYmazgUF3d2AJsKlu9fOBT7j7MIC7P9qocTNbYWbrzGzdEINF9F9Eqqew3FHmiEgDyhwRSUnfr0S6XPIjeMxsGfBZ4GBgKzA7W2zrgVOB1WShtNDMNpKNJq8zs5XuvrrV53H3VcAqgPm2qLrX5RORKUuRO8ocERmlzBGRlPT9SkRGJR/gcff1ZIcI/hA4g+wQwQ+5+215yQVmdjFwF/AI8CJ3f1ddE1cAv2NmV44eQthslFlEBJQ7IpKWMkdEUlLmiMiojpyiZWaLgW3uXgOW1oXPqLOAa8hmer9qzLpPAT8HNprZBuA1RfdXRKpPuSMiKSlzRCQlZY6IQOcmWd4CvDi/f0qD9S/N736wwbphsonD3lFkH0VkelHuiEhKyhwRSUmZIyLQ2UmWRURERERERESkDTpyBE9HeEXnASu437UdO2MP8FqofOSBh2LNDw23XNvT2xtqe3CBherZHbs6gA0Ht83++4TqewZb74/Pmx1qW+QXLPg5qcXe9z2zZsaaXzAnVD+4OPbeH+gfaLnWh/aE2o469JtDsQc89kSovHf/xbH2I39/RkZCTQ8/GPvbIDLKh4Ofk+B+lO8p9nMe6U/4tQbNuzf2ue2784FQ/ebXHBWqf9q2WS3X1mbEvsLY3feE6qVFVf1+FVT0Z9EtdsxF+AiN6L5dQO3JHaH6fb829szBCfTFPus/eefRsfYDG3Pef8f6UvR+YzM6gkdEREREREREpOKSD/CY2UozO9fMLjSz94xZd4yZXW5mPWa2tm75OWb29dR9FZHqU+aISGrKHRFJSZkjIqM6cQTPycC1wNnA1WPWnZkvOx64JXG/RGR6UuaISGrKHRFJSZkjIkDCOXjM7BLgfOBwYC1wJHCemX0JuBK4DDgE2AzMA2pmts7dl49pZxHwaeAIYCewwt03pnodIlINyhwRSU25IyIpKXNEZKxkR/C4+8XAm4HPAs8FNrr7Ce7+AXf/vrsvAzYBxwJXAC8cGz65vwBucvcTgD8BPtfo+cxshZmtM7N1Q8QmyxWR6lPmiEhqKXNHmSMi2tcRkbFSn6J1IrABWArcXr/CzGYDg+7uwBKyMGrkDOBfANz9e8C+ZjZ/bJG7r3L35e6+vJ8ZbXwJIlIhyhwRSS1J7ihzRCSnfR0R+YUkp2iZ2TKykeWDga3A7GyxrQdOBVaThdJCM9sIHAasM7OV7r46RR9FZPpQ5ohIasodEUlJmSMijSQ5gsfd1+eHCP6Y7BDB7wHnu/syd9/l7hcAnwTeArwN+Od8XaPw+T7wWshmfwe2uvsTKV6HiFSDMkdEUlPuiEhKyhwRaSTZKVpmthjY5u41YKm73zam5CzgGrKZ3q8ap6n3AyflI9F/A1xUQHdFpOKUOSKSmnJHRFJS5ojIWMmuouXuW4AX5/dPabD+pfndDzZYtwZYk99/FLiwqH6KyPSgzBGR1JQ7IpKSMkdExko2wCMlVRsptHnfs6e4tnfHZu/f75bdofrhhzaH6vvmzw3V29BwqN4XL2q59mcv2y/U9iE/+VmsL8NDoXo8Vj4teEVfdLDfI3feXVBHcjfvCJXP3GdBqH5kpMAMDG7LgS2x12rz54Xq/YntofrBZx/Wcu0DZ8Ym2zzk/Q+F6kV+oehsDbbvg8ErCfX0xuojemNtL1j3YKjeF8Qy56DvPBKqf/DcfVuu7RmK/Z4WXxf7yuPRvw0V/ZMvLSo8d2LvNx8J5kiB/Y9+VkYej535ZwMDofqj//GuUP2vX3lzy7Vf/Mb5obZ7rropVN8uqa+iJSIiIiIiIiIibaYBHhERERERERGRimvbAI+ZrTSzc83sQjN7z5h1x5jZ5WbWY2Zr2/icbzCzj7arPRGpDmWOiKSm3BGRlJQ5IhLVziN4TgauBc4Grh6z7sx82fHALW18ThHpXsocEUlNuSMiKSlzRCRkypMsm9klwPnA4cBa4EjgPDP7EnAlcBlwCLAZmAfUzGyduy83szcALwMWAAcBn3f3v8jbfR3wNmAA+BHwe+4+YmZvBN4DPAZsAIKz24lIlSlzRCQ15Y6IpKTMEZHJmvIAj7tfbGZfAF4PvANY4+6n15Usyw8bPA34NHCpu99at/55wHHATuB6M/sGsAN4NXC6uw+Z2ceB15rZFcBfACcBj5MFXMPpqc1sBbACYCazp/oyRaQklDkikloZc0eZIzJ9lTFzQLkjUgXtukz6iWSjvUuB2+tXmNlsYNDd3cyWAJvGPPYKd38kr/0KcAYwTBYy15sZwCzgYbLDFNe4+5a8fjVwdKMOufsqYBXAfFukixeKTC/KHBFJrVS5o8wRmfZKlTmg3BGpgikN8JjZMuCzwMHAVmB2ttjWA6cCq8lCaaGZbQQOA9aZ2Up3X503MzYcHDDgcncfO5nYhVPpr4hUmzJHRFJT7ohISsocEZmKKU2y7O7r3X0Z8GPgWOB7wPnuvszdd7n7BcAngbeQne/5z/m61XXNvMDMFpnZLOBC4AfAd4FXmNn+APn6Q8nOFT3bzPY1s37glVPpv4hUizJHRFJT7ohISsocEZmKdkyyvBjY5u41M1vq7reNKTkL+BzZ+ZpXNWjiOuDLZKPUn3f3dXm77wW+Y2Y9wBDw++5+rZm9n2yysceA9VPtv4hUizJHRFJT7ohISsocEZmsdkyyvAV4cX7/lAbrX5rf/WCTJu5z970ODcxHoVc3WP4Z4DOT7rCIVJoyR0RSU+6ISErKHBGZrHZNsizSmBc3/1rPgnmheqsF+xLs+8hP7g62XwuV28BAy7XzT4/13WbOiNUPBc/u3BUrl87pmTMnVF/buTPW/ty5oXrrjb3XrL8/VN+7z4KWa2uPPxHrS+AzOxkj9z8Yqu9dvF+ovv+x3S3Xzn4w+FqzCT6Lo6k/paSsp/X3vvUFd9OHh0PlI1u2hup791kYqieYx/PvaT2PHzy1N9T2/sceFaq3O38eqmdHrFw6rCf2/gnvs/fG2vfod5RAjgD0zJzZem30cz4Q+5xT8Gvd8awDQ/VffVHr2+bBV80KtX3Q94t9nzXb1+noAI+7f5ZsdDtukQAAIABJREFUEjERkcIpc0QkNeWOiKSkzBHpblOaZHkyzGylmZ1rZhea2dhZ3I8xs8vNrMfM1tYtX25mH0ndVxGpPmWOiKSkzBGR1JQ7IjIq+QAPcDJwLXA2cPWYdWfmy44Hbhld6O7r3P1tyXooItOJMkdEUlLmiEhqyh0RARKeomVmlwDnA4eTzdJ+JHCemX0JuBK4DDgE2AzMA2pmts7dl5vZOcA73f0lZvY84MPATLKZPd7o7ptSvQ4RqQZljoikpMwRkdSUOyIyVrIBHne/2My+ALweeAewxt1PrytZlh82eBrwaeBSd7+1QVN3AGe6+7CZPR/4a+DlBXdfRCpGmSMiKSlzRCQ15Y6IjJV6kuUTgQ3AUuD2+hVmNhsYdHc3syVAs1HjBcDleY0DDafuNrMVwAqAmcxuT+9FpGqUOSKSkjJHRFJT7ojILyQZ4DGzZWSzuR8MbAVmZ4ttPXAqsJoslBaa2UbgMGCdma1099VjmvsgcKW7v8zMDgPWNHpOd18FrAKYb4t0wVSRLqLMEZGUlDkikppyR0QaSTLA4+7ryQ4R/CFwBtkhgh9y99vykgvM7GLgLuAR4EXu/q4mzS0A7s/vv6G4XotIVSlzRCQlZY6IpKbcEZFGkl1Fy8wWA9vcvQYsrQufUWcB15DN9H7VOE19CFhpZjeR/hQzEakIZY6IpKTMEZHUlDsiMlbKSZa3AC/O75/SYP1L87sfbLBuDfmhgu6+Fji6bvV729xVEZkGlDkikpIyR0RSU+6IyFjJjuAREREREREREZFi6BC81Mxi9d5l85dFts/C+aGmB376cKh+OFQNeC36iBAbGGi5ds9/LQ61vevMfUL1vbuDr/W7sXLpnNrOnbEHBDOq9uSTsfYt+P8Qjz0eKu878ICWa31kJNS279oVqufmZhc3aQ8f3BOqH1ows+XaR5fFts2BT39aqL4W/L0SfJuJJBPItJ7F+4Wari2YG6r3hzaH6ocf3hqqj5qzfXvLtc/YeVSo7R9fPCtUf8wfxerZESuXDqvF/maFRfddarG/z74nth/ugf7YvDmhtmv33Bfry1DsG5b19obqZw80vABcU9ue1/r+yO79Yvu8vYsWhup9Z3C/sUnu6AgeEREREREREZGK0wCPiIiIiIiIiEjFaYBHRERERERERKTiNMAjIiIiIiIiIlJxGuAREREREREREam4aXsVLTNbAawAmMnsDvdGRKY7ZY6IpKTMEZHUlDsi5Tdtj+Bx91Xuvtzdl/czo9PdEZFpTpkjIikpc0QkNeWOSPlN2wEeEREREREREZFuUfkBHjP7bzN7eqf7ISLdQZkjIikpc0QkNeWOSHVVfg4ed39Rp/sgIt1DmSMiKSlzRCQ15Y5IdVX+CB4RERERERERkW5X+SN4Kse90z0ot8D2Gbnz7ljbZsHOBBX8u609+WTLtQf8y8ZY2886IlR/yidvDNWvOSFULp1UdEZF2/eRYvqRG35oc6Hth/T0xuq9Fiof2bo1VD9w/e6Wa5+5Lpav/rT9Q/UnfePeUP13nh0qF0nGh/a0XDt8732xxmMfk+LVisvvGdf/JFR/+MhRofpXXXNzqP7bx4TKu1Pwb5z1t/411QcHY30p+DuBDw/FHlD0d5TAZ7F2dyxIbKA/Vr/k8FB9z45dofravQ+E6hc8tKX12jsODrU9tPQZofp7XxC8Mt37PtdwsY7gERERERERkf/P3p3HS3bWdR7//O7t7nQ6SXd2EgNkIyRAiE0IEPYgagSEgRFExZHFmbgMg6MjCG6juESB0RFwoWHYxlGD4IIbgpAQkMTQSTqdzSAmQdasnbXXe+9v/qhzpXL71u167q1z6pxbn/frVa+uqvOrp546VfW9p54+5zmSOs4BHkmSJEmSpI6rZYAnIk6KiF0Rsa26PRsR2/oub6zuvyQitvY97pyIuKS6fl5E3BsRV0fETRFxaUR8d1/tT0bEv0XEO+t4DZK6xdyR1CQzR1KTzBxJw6hzDp5/zczN1fVdfdcXOjYinpeZf7fIss9k5ncDRMRm4C8iYldmfjIzfzsidgDn1NB3Sd1k7khqkpkjqUlmjqQlteEQrbcCP3egoszcBrwZeG3tPZK02pk7kppk5khqkpkjTaimBngOXrAL4cv7ll0G7I2I5wzRzlXAGcM8YURcEBFbI2LrPgpnVpe0GjSaO2aONPHMHElN8veVpP00dZr0pXYhBPhV4OeBnzlAO0OfQy4ztwBbADbGkZ6bXJo8jeaOmSNNPDNHUpP8fSVpP204RIvM/BRwMHDuAUqfANxYf48krXbmjqQmmTmSmmTmSJOpFQM8lV8F3jBoYUScBfwC8LuN9UjSamfuSGqSmSOpSWaONGGaOkTr4PlT+lU+lplv7C/IzL+NiDsWPO6ZEXE1sAG4HXhdZn6y5r5KWh3MHUlNMnMkNcnMkbSfRgZ4MnN6wP3nLbj9xL7rlwCbau2YpFXL3JHUJDNHUpPMHEmLiczRz48VEY8APgfcdYDJv1byHD8J/Cjwkcz82QPU3gF8aZFFRwN3FjxtSX2dbXe9vk19aVt9m/oyyvoTM/OYgnaKtSl3xpQ5dde3qS9tq29TX9pWb+as7DkmOXPqrm9TX9pW36a+tK3ezPlm7Shyp03vbdvq29SXrte3qS+jrF88dzJzYi/A1rrq62y76/Vt6kvb6tvUlybqJ+3StvXvZ9N1M2nrZtIubVv/bapvU1/aVt+mvrSt3swZ7/qcpPo29aXr9W3qSxP1bZpkWZIkSZIkScvgAI8kSZIkSVLHTfoAz5Ya6+tsuzP1EfHAgtuvAu4fUduXRMQ5iyz6QkR8MSIyIo5ebvtjqm9TX5qonzRtW/+r8rM5htzZEhH/LyJuiojrIuK9EbG2tO1VWt+mvkyitq3/NtV3OXMAdkfENRGxPSI+HBGHLqf9EdROWr2Zc2B+1kZT37bM2VItf/vC5y9pf0z1bepL7fW1TLIszYuIBzLz0L7brwLOyczXjqDtS4CfzsytC+5/ArADuKR6rpJJrCR13Jhy5/nA31U3/wi4NDN/f6XPJ6n9xpQ5GzPzvur6bwG3Z+ZvrPT5JLXfODKnWnYO8BPAS/qfX+0y6XvwaIwi4piI+EhEfL66PL26/8kRcVlEXB0Rn4uI06v7D46IP4mIGyPiz4GDF2s3M6/OzFubeyWSuqLG3PnbrABXAA9v7EVJaq0aM2d+cCeqGv/HVlJtmRMR08BbgTc09mK0LGvG3QGtegdHxLa+20cCH62u/w7w25n52Yh4JPD3wGOAfwaemZkzEfHtwK8D3wP8GLAzMx8TEWcBVzX2KiR1ydhypzo06z/R+x8uSZNhLJkTEe8Dng/cAPyPUb8oSa01jsx5LfDRzPx6b1xZbeUAj+q2KzM3z9+Y34WwuvntwGP7QmJjdQz5JuADEXEavf+Rmp/L4lnA2wEyc3tEbK+/+5I6aJy583v0Ds/6zCheiKROGEvmZOarq/9VfwfwcuB9I3tFktqs0cyJiG8BXgacN/JXopFzgEfjNAWcm5m7+++MiHcCF2fmSyLiJHpz6UjSKNSWOxHxP4FjgB9ZeTclrRK1butk5mxE/Am9wyYc4JFUR+Y8AXgU8MVq4GhDRHwxMx81kh5rpJyDR+P0ceC/zd+IiPmR6E3AV6vrr+qrvxT4gar2TOCs+rsoaZWpJXci4j8D5wPfn5lzo+2ypA4beeZEz6PmrwMvonf4hSSNPHMy828y87jMPCkzT6J3SJeDOy3lAI/G6XXAOdE7xecNwI9W978FuDAiruahe5n9PnBoRNwIvBm4crFGI+J1EfEVepOcbo+I99T2CiR1TS25A/wB8DDgsojYFhG/WE/3JXVMHZkT9A61uBa4Fji+qpWkurZz1BGeJl2SJEmSJKnj3INHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOc4BHkiRJkiSp4xzgkSRJkiRJ6jgHeCRJkiRJkjrOAR5JkiRJkqSOWzPuDjQpIt4+RNl9mfnztXdG0qpn5khqkpkjqUlmjtQ+kZnj7kNjIuJLwC8eoOyNmfmYJvojaXUzcyQ1ycyR1CQzR2qfidqDB/jtzPzAUgURcURTnZG06pk5kppk5khqkpkjtcykzcEzc6CCzPzfTXRE0kQwcyQ1ycyR1CQzR2qZSRvgec24OyBpopg5kppk5khqkpkjtcykDfBogIiYjoifHHc/JE0GM0dSk8wcSU0yczQukzbJ8gywc7FFQGbmxoa71CoRcUVmPnnc/dDoVcc/nwasn78vMy8dX48mg5mzNDNn9TJzxsPMWZqZs3qZOeNh5izNzFnd2po7kzbAc3VmPmHc/WiriPhtYC1wEfDg/P2ZedXYOjVmEfGGzHxLRLwD2O/LkpmvG0O3ikTEfwZ+Ang4sA04F7gsM79trB2bAGbO0syc/Zk5WgkzZ2lmzv7MHK2EmbM0M2d/qyFzoN25M2ln0dLSNlf/vrnvvgTG/kEdoxurf7eOtRcr8xPAk4DLM/M5EXEG8Otj7pMEZs5izBypPmbO/swcqT5mzv5WQ+ZAi3Nn0gZ4/nSxOyPiO4HXZ+Z3NNyfVsnM59TZfkScDTyDXrD9YxdGrzPzr6p/lzwFZJMiIoBXAKdk5psj4pHAcZl5xYCH7M7M3RFBRByUmf8cEaePqC9vH6Lsvsz8+VE8XweZOUswc/Zn5hywL2bO0sycJdSdOdC93Glj5kBx7pg542PmLMHM2d8qyRyoKXdGkTmTNsny5RHxhYh4ICL+MCIeHxFbgQuB3x9358YtIh4WEf8nIv6uuv3YiPjhEbX9i8AHgKOAo4H3RURn/hhGxCci4vC+20dExN+PqTu/BzwV+P7q9v3A7y5R/5Wq738BfCIi/hL40oj68h+AKw9w+Z4RPVcXmTlLMHMGM3MGMnOWZuYsoc7MqdrrbO60LHOgLHfMnPExc5Zg5gzW8cyB+nJn5ZmTmRNzAa4GzgMOAl4MPAC8doz9eTTwSeC66vZZwM+PsT9/B3wvcE11ew1w7YDanxjmvr5lNwHr+24fDNw07s9EwbrZttjnaUx9uWrh88+/Z0M89tnAi4B1I+rLfx9FzWq9mDkH7M/QmVMtHzp3zJyR9sXM6cjFzDlgf2rLnGpZZ3OnTZlTPfeycsfMafx9alXmVH1qTe6YOUuum1WROVXdyHJnFJkzaXvwkJmXZOaezPwL4KuZ+c4xdufdwJuAfVXftgPfN8b+HJ2ZHwLmqv7MALMDal+5yH2vWqLtr9E3wzi9PwRfXUYfx2W22lUPgIg4kUUmBluJiDg0Ig4donRfREzPP39EHEP1ni1ob2P175HzF+Ba4LPAMM9zQJn5v0dRs5qZOUsqyRwoyx0zZwhD5o6Z0yFmzpLqzBzodu60KXNgiNwxc9qhZZkD7codM2ewzmVOdX+tuTOKzJm0OXg2RcR/7Lu9pv92Zv5Zw/3ZkJlX9A75+3czDfeh34MRcRTf/GCfC9zbXxAR3w/8AHByRHy0b9FhwN1LtH0vcH1EfKJq/zuAK+aPM8z2z5j+c8BnI+LT9E79+EzgglE0HBGPBz4IHNm7GXcAr8zM6wY85O3AnwMPi4hfA14KLLY75h8B301vV76s+j0vgVNG1P/nAP8NmD/u9EbgnZl5ySja7zgzZ2kHzJzq/uXkjpmzhMLcMXO6w8xZWp2ZA93OnTZlDgyXO2bO+LUtc6BduWPmDNbFzIEGcmelmTNpp0l/3xKLMzNf01hngOp4zNcCf5qZZ0fES4EfzsznNdmPvv6cDbwDOBO4DjgGeGk18j1fcyJwMr1ja9/Y9/D7ge3VyPRibS82Iv3vsmUTbS0mIo6mdwo86M2YfueI2v0c8HOZeXF1+zzg1zPzaUs85gzgudXNT2XmjYNq6xQRLwDeSe/sAFfRC7mz6QXiazPzb8fRr7Ywcw7YnwNmTlVXnDtmzgHbLsodM6cbzJwD9qe2zKke1+ncaVPmVDVjzx0zZ2ltyxxoV+6YOUszcxbtw4ozZ6IGeJYSEQ/LzNsafs5TgC3A04AdwC3AKzJzRRM0RcSHMvN7I+JaHrqrW9AL27OWeOwaeqOFQe8Yzn0r6cuCttfROy6WUbfdhIg4ATiRvj3fMvPSEbR7TWZ+64HuW7D8gDPmVzUDLfaYUhFxCb1jg69ZcP9ZwDsy89krfY7Vysz598eaOQPUlTlV20W5Y+Z0n5nz74+tLXOq9jubO23KnGr5krlj5rTbODKnet5W5Y6ZM1jXMqevZqCV5s4oMmfSDtF6iOjNfP099HaJewzwLQ0+9zTw45n57RFxCDCVmfePqPmfqP797sI+/Vfg/2Xm9dXtIyLi+zPz9xapPZfeiPRjgHXANPBgZm4c0PZ59GZ5v5VewD0iIl456EscEacCX8nMPdVjzwI+mJn3lLymUYmI3wReDlzPN4/HTGC//vetx3uq20cAi67Hys0R8QvA/61u/yBw8xJ9+UXgZcBH6K3L90XEn2bmry4o/V/Vv+uBc4BrqvqzgK30ZopfqeMWBhD0jneOiIeNoP1VxczZr09DZ061fOjcmaTMqepryx0zp7vMnP36VFvmVPXn0dHcaVPmVO0NkztmTsuMM3Oq529V7pg5g3U0c6D+3Fl55mQLZtFu8kJvdvHvAz4KfBm4h97s71Nj6MvlNbY9DVxc+JihZzOn9wF+FL3Z86eBVwMXLtH2lcDpfbcfDVy5VF/oDUA+CvgC8Fbgb8f4ubkJOGjU67FadgS94z6vqi6/AxxxgL4MPWM+8GfA4/tunwl8eETrZan3cOCySbqYOUs+pvS7MnTuTFLmLHNdDp07Zk63LmbOko+pLXOq+s7mTpsyp68/Q+WOmTPeS5syp+pPa3LHzFly3XQ2c6rlteTOKDJnovbgiYg/ojeB08fpjY5+Cvhijm+StKujN5HWnwIPzt+ZI5iQLDNnI2IuIjZl5n6TeQ0wHRGR1SeoGgVft8RzfDEipjNzlt4o59X0Zq1fzNrMvKnvsV+IiLVL9GUuM2ci4iX0dkd7R9X+uNwMrAX2DFFbuh53ACWToM3PmL+7un2gGfNPz8xr+57vuoh4TMHzLeXUeOhkcPOCEU1u2GVmzgEVfVeq5xk2dyYpc6De3DFzOsLMOaA6Mwe6nTttyhwoyx0zZ0xamDnQrtwxcwbrcuZAfbmz4syZqAEe4LH0jsW8Ebix+pKOcxKi9cBdwLf13Zf0RgRH4QHg2ujNrN4fcIM+7B8DLoqId1W3f6S6bzE7o3fM57aIeAvwdWBqib5sjYj3AH9Y3X4FvVHqQfZFb0b5VwIvrO5bKrDqtpPea/0kfUE0YF0OtR4j4q9Y4nSAmfmiAYtKZ8zfvsi6385o/Icllr1tRM/RZWYOI8scKMudScocqDd3zJzuMHMYW+ZAt3OnTZkDZblj5oxP2zIH2pU7Zs5gXc4cqC93Vpw5EzfJcvRmx/5+esf83Ulv0qszcwyTgNUtBsysngNmVI+IKXqnp/v26q5PAO+pRpAX1p4I3EZv5PQngU3A72XmFwe0fRDwX+lNXAXwmap+0VHbiHgs8KPAZZn5xxFxMvC9mfmbi9XXrWRdDrseI2LJSbIy89MlfRnUp4hYD/wY8KzqrkuB38/M3Qsfq9Ezc0aTOVX90LkzSZlT1deWO2ZOt5g548mcqr6zudOmzFmqP4v1y8wZr0nKHKjnN0FfvZnTgcyp6lubOxM1wBMR52bm5X23n0gvkL6X3oRTA0+bVlN/TqF3POC59EYKLwP+e2be0mQ/pOWKiIsZPEqemfncAcsmgpkjjZaZszQzRxotM2dpbcucqg/mjjprFJmz1C5fq9FDZtnOzCsz86fpnZ7tjWPozx8BHwKOpzfL/J8CfzKqxiPiir7rLxtlfZ1tL6e+bl1eNzWvy58GXr/g8hHgVGDRGf8njJkzpvqWfU+KuS4HMnOWZuas4vo6te211plRhcycpbUtc6BFudOm78ly6uvUttfaonW58szJMcyaPa4LcNW4+7CgP9sXue+aEbT7OeBd9E6Zdwa9WdgHvvaS+jrbXk59A+9RZ9dN0+sSeDbwD8BngeeN6z1r08XMab6+7d8T1+VI15WZs/86MXNWYX3N71GrXmudGTWCdWXm7L9OWpU5VZ/Gnjtt+p4sp77m96dVr7XN63K5mTNph2jdQ+/4uEXl4EmXRt2PI6urP0NvYrI/obcr1svpnb5t0Ezpw7YfwOOBv6F3fOJp1e0/AD6dmX+33Po6215Ofd26vG6aWpcRcT7w8/QmSPu1zLx4FO2uBmZO85/ltn5PhuW6PDAzZzAzpxt/b9uUO217rXVm1HKZOYO1JXOqvrQmd9r0PVlOfZ3a9lrbuC5XmjmTdhatO4D/Ne5OAFfSC5yobv9I37Jk8KnwhvVeemF7X2a+BiAirgH+jt6pDBd+8Erq62x7OfV16/K6qX1dRsTngWOAt9I7xpmIOHt+eWZetdLn6Dgzp/nPcuu+J4Vcl0swcw7IzOnG39s25U7bXmudGVXMzDmgtmQOtCt32vQ9WU59ndr2Wlu1LkeROZM2wPNADpg5ezER8T4WmeRo/s1crsw8eSWPH8KF9D5gx0XEP9Ib/XsYcCS9XcpWUl9n28upByB6M5n/MPA4eqdHBFb+XhX2p23rZlnrstCD9E4X+dLq0i956CkqJ5GZ0/xneZIyp+56M6d7ijIH6smdCcucJuqB2nKnba+1zoxaDjNnaa3InOrxbcqdNn1PllMPmDkjqi+18swZ9liu1XAB/qyw/nv6Lq8APgy8fYT9WQu8rmr3w8BrgbUjbP/q6t8NwE30Jm36y1HU19n2Muv/FPgV4F+BVwIfB35nHOuyheumqN7L6C5mzvg+y5OUOV1fl15GdynNnOoxteXOJGVOQ/W15U4LX2ttGeVldJe2ZU7Vfmtyp03fk2XWmzkjqm/yMvYONPpi4UnAcX23fwj4S+DtwJFDPH4K+NwI+/Me4AP0RuK+DXgf8J4Rtv+MvusH/MCV1NfZ9jLr579k26t/1wKXj2NdtnDdFNUXrpc39F1/2YJlvz7K5+rixcwZX/0kZU7X12XhejFzll4/K8qc6jEjy51JypyG6mvLnRa+1toyqnC9mDlLr59WZU7VXmtyp03fk2XWmzkjqi9YLyvOnEk7Tfq7gL0AEfEs4DeADwL3AluGePxpwLEj7M+TMvOVmfmp6vJqekE5Kuf1Xf/eEdfX2fZy6vdV/94TEWcCmxjte1XSn5LaNtaX+L6+6wuPbf6uET9XF5k546uvuy9typy66+vuSwkzZ2krzRwYbe5MUuY0UV9n7pT2pU31pW2XMHOW1rbMgXblTkltG+vNnNHVD2vFmTNpAzzTmXl3df3lwJbM/Ehm/gLwqIXFEXF/RNw3fwH+CnjDCPszGxGn9j3fKcDsShuNiJ+JiKfy0OP2LhtFfZ1tL6e+z5aIOILejOMfBW4AfnOIxy2py+tmBeuyRAy4vtjtSWTmNFw/SZlTd72Z00lFmQO1586qz5wm6vuMPHfa9lrrzKhlMnOW1rbMgRbkTpu+J8up72PmrLB+GVacOZM2yfJ0RKzJzBngucAFfcsWWxeb6B0benJmvjkiHgkcN8L+/DRwcUTcXN0+CXj1CNr9Z+BlwCkR8Znq9lERcXpm3rTC+jrbXk79vP9L71jek+jtlgm9Ca/2ExFB7309pf99zcwrVtiftq2b5a7LEjng+mK3J5GZ0/xnuXWZA0W5M4nrsoSZs7TSzIF6c2cSMqeJ+nl1bOu07bXWmVHLYeYsrW2ZA+3InTZ9T5ZTP8/MWXl9qZVnzqiOF+vCBfg54B/pHRt6NRDV/Y8C/nGR+t8Hfhe4sbp9BPD5EfbnZcBG4CzgF+mdVu3sEbT7bHoznV8FTANnAl+idwzqfse4ltTX2fZy6vse9zHgInr/A/A/5i8Daod+X7u8bpa7Lgs/a7PAfcD9wEx1ff72vrq/022/YOY0/llu6ntCQeaUvLeTuC4LP2tmztLrpyhzSj6by+zPqs+cJur7HjfybZ22vdaS+uWux8LPmpmz9PppVeZU7Y09d9r0PVlOfd/jzJwRrcuCz9mKM2fswdD0BTgXeAlwSN99j17siw9cVf17dd991yxS96Hq32uB7X2Xa6kmpRrQl/kJq54BXAy8APinAbW/Ocx91f2/DnyS3vGvv0Pv9HY3LNGPoevrbHs59X2Pu67gMzDU+9r1dbPcdelltJc6Mqe6vzh3SjKnqhsqd9r0WW5j5pS8t5O4Lr2M9lKSOdWy2rZ1JiFzmqjve9zIt3Xa9lrrzCgv9VzalDlV/dh/X7Xpe7Kc+r7HmTkjWpdNXsbegUZfbPXBG7YG+Cd6I3PzH9hj+j+0fXXHV/+euNhlieean5n8QuAH+u8bpu9LhVu1/BrgHHq7S94BfBb4q1HU19n2Muu3AI8f8nMw1Pu6itZNUX2d36lJu9SVOdWy4twpyZxB/V8qd9r0WW5T5pS+t5O2Luv8Tk3aZTnrZ9jPppnTivratnVa+Fpry6i6v1OTdGlb5lT1rfl91abvyTLrzZwR1df5ndpv+Uo70aULsIuHjgAvvFwL/FuS7vmKAAAgAElEQVRf/SvoTSj1FeDX6J3j/mUj7M9f05t9/mbgcOAgFox0Aj9W9evBBX29BfjDA7T/lr7r82F39Cjq62x7mfU30JvF/yYO/D+Kxe9rx9dNUf2ANo4bcH/Rd2rSLl3MnKpuWbnTps9ymzJnOe/tJK3LAW2YOcu4LGf91Jk7k5Q5DdXXtq3TwtdaW0Yt8Zz75Y6Zc8B11qrMqdpvze+rNn1Plllv5oyofkAbtWTO/HGSEyEiThyibDYzv9L3mDPoTRoWwCcz88ZF2r2fxSc9CiAzc+OA/mygd7qzazPzXyLieHqjpB/vq9lE7xjGC4E39j38/vzmrPUHFBHfmpnX1FFfZ9vD1g96bzPzSwPqD/i+rqQ/y6ltY33f4/4mM1+wyP3F36lJUlfmVHXFuTNM5lR1K86dNn2W25A51WOWlTurfV0OeJyZswzLXT91betMaubUVd/Utk4bXuty65ebOdVj98sdM2dpbcuc6nGt/H3Vpu/JsPVmTj31fY+rJXMmaoBHkiRJkiRpNZoadwckSZIkSZK0MhM9wBMRF9RVX2fbXa9vU1/aVt+mvjRRP2natv79bI6mvk19aVt9m/oyidq2/ttU36a+tK2+TX1pW72Zc2B+1kZT36a+dL2+TX1pon4kk1l19QJsrau+zra7Xt+mvrStvk19aaJ+0i5tW/9+Nl03k7ZuJu3StvXfpvo29aVt9W3qS9vqzZzxrs9Jqm9TX7pe36a+NFE/0XvwSJIkSZIkrQYTMcnyujgo13PIfvfvYw9rOWi/+487c+ei7dx79yybjpze7/5v3Hzk/m3P7GTtmg37N7Jz96JtD+rLIF2uX6o2Iva7by97WLdI/aDPbptea2l9m/oyyvr72XFnZh4zdEMdN33oIbnmqCP2u3/2gQeZPnT/LDrorgGf5X0Psnbt/vWxZ++i9XvndrNuav1D71wi4vfmLtbFwQvuXfwBi7YNMLX4/xPsndvFuqmFbUPOzC5avy93szb2b3+xTADYm7tZt0j9wFwY0P4gi9Uv3pPJyqjSejOnGWsPOiQPOmSRbZE9D7D2oEP3u3/6wQEZMruLddP7f2/3bVy3aP3M7gdZs/6hGbX27l0D+7no93btmsH1sztZN71gW2qpTJvbybqph9bnvn2L1g7MnEFtD/qeD+jQvtzD2ti/PgZm5v4Zm7NzA3qzOr+3Xag3c75pXazPg2P/bZTB35X9Lfm3ecP+9w/cLhqwbbFohgB54oDv7b27WLtp/wzkCwNyZMDnYeC2y4B1w2K/fwZs5wDk3P7Z0KbvSdvq29SXUdYPyp3Bf1VXkfUcwlOmv3Po+jd8dFtR+2996fcNXZtXX1/UdrEBgTJQ6QBfze1PrR/+B9jc7sUHy9Q+/5AfHnjq6NVozVFHcNzP/sTQ9ad9cE9Z+1/48vDFs4tv9Aw0V/adjQ2LbAgtYXbHPWXtryn7M5UzM0X1Ja83pst2ejWjxmfSMuegQ47krOcOnzmbtn6tqP1vnP/woWuP/cOyM8VOHXdsUX3pdsXsV75eVF/6PS/NnKkNi/zn3wCz991X1Hax0m26ukXBup8r/NtWs0nLHICD4xDOXf/8oeuXGrBczNy3Pmbo2rW33VvU9swfFG4rfEdZjkytW1vW/oCB30Hmdi6+M4Imy6Dc8RAtSZIkSZKkjuv8AE9EfG7cfZA0OcwcSU0zdyQ1ycyRuqvzAzyZ+bRx90HS5DBzJDXN3JHUJDNH6q7OD/BExAPj7oOkyWHmSGqauSOpSWaO1F2dH+AZJCIuiIitEbF1H2UTmEpSqf7MmX3gwXF3R9Iq95DtnD3+FpNUv/7c2evvK6mVVu0AT2ZuycxzMvOcktOQSdJy9GfOYqdCl6RResh2ziKnQpekUevPnUVP9y1p7FbtAI8kSZIkSdKkcIBHkiRJkiSp4xzgkSRJkiRJ6rjOD/BkpgeeS2qMmSOpaeaOpCaZOVJ3rRl3B5oQU1NMHbx+6Pr/dc6zitr/8Sv+fOja33306UVtl5o++uii+tm77i6qj7VlH5ncN1NUP7d7d1F9raamy+rnZuvphzrnoH/byaN//PPDPyCzqP2//dq2oWu/68QnF7WdM/uK6rnvvrL6QrmnPWfpyMJVIzVlaseDHPJnVwxdX/aXGa78pb8auvb8LZuL2p67+dbC3tSr+HteuK0wW3NmFin821O7dDuqSzKz1u32v/vw+4auff4JZxe1veZVDy+qnyncxp/b7WdZ49P5PXgkSZIkSZImXacHeCLiVRHxznH3Q9LkMHckNcnMkdQkM0fqtrEO8ERPpweZJHWLuSOpSWaOpCaZOdJka/zLHxEnRcRNEfFB4DrgFyLi8xGxPSJ+ua/uLyLiyoi4PiIu6Lv/1RHxhYi4Anh60/2X1D3mjqQmmTmSmmTmSJo3rkmWTwNeCWwEXgo8GQjgoxHxrMy8FHhNZt4dEQcDn4+IjwDrgF8GngjcC1wMXD2OFyCpc8wdSU0ycyQ1ycyRNLZDtL6UmZcD31ldrgauAs6gF04Ar4uIa4DLgUdU9z8FuCQz78jMvcBFg54gIi6IiK0RsXVvtujMTJLGpdbc6c+cfbTnzE+SxsbMkdSkRn9fmTtSO41rD54Hq38DuDAz39W/MCLOA74deGpm7oyIS4Dhz3MOZOYWYAvApumjW3YeSEljUGvu9GfOxjjSzJFk5khqUqO/r8wdqZ3GPQHX3wOviYhDASLihIg4FtgE7KjC5wzg3Kr+n4BnR8RREbEWeNlYei2py8wdSU0ycyQ1ycyRJti49uABIDM/HhGPAS6LCIAHgB8EPgb8aETcCNxEbzdCMvPrEfFLwGXAPcC2cfRbUneZO5KaZOZIapKZI022xgd4MvNW4My+278D/M4ipc8b8Pj3Ae+rpXOSViVzR1KTzBxJTTJzJM0b9yFakiRJkiRJWqGxHqLVmLVriOOPHbp87stfK2r+Ha95+dC1ax63q6jt2Y0HFdXntTcX1a856RFF9WTZfGpzt99ZVL/3KWcMXbvm4quK2i7tO3OzZfVSJQ5ez9QZjxm6fm7bDUXtf9eJTx669vaPnFzU9rH/8V+L6nOu7Hs1vfHQsvb37iuqjzVlf9Zydvjveawta3v2nnuL6qXliqkppg4d/rs1d//9Re2ff8IThq5dc/zDitq++YJTiupP2VK2nZOHHVJUP3tkWf3U9beU1W/aOHTtzFe+WtR25/UOJ6pH6TaghlPynhW+B89/+BOHrp0+6oiitn/t0o8U1b/xtKcX1ce6dYX1a4vqZ++9r6i+iN+VznMPHkmSJEmSpI5rfIAnIh4YUTubI+L5o2hL0upl5khqkpkjqWnmjqR5Xd6DZzNgAElqipkjqUlmjqSmmTtSx9U6wBMRfxERV0bE9RFxQd/9v13d98mIOKa6b3NEXB4R2yPizyPiiOr+SyLinOr60RFxa0SsA94MvDwitkXE8JPgSFq1zBxJTTJzJDXN3JG0lLr34HlNZj4ROAd4XUQcBRwCbM3MxwGfBv5nVftB4Gcy8yzg2r7795OZe4FfBC7KzM2ZeVGdL0JSZ5g5kppk5khqmrkjaaC6B3heFxHXAJcDjwBOA+aA+cD4Q+AZEbEJODwzP13d/wHgWSt54oi4ICK2RsTWvbNlZ66S1FntyJyZnStpSlJ3tCNzcvdKmpLULa3InX3sWUlTkmpS2wBPRJwHfDvw1Mz8VuBqYP0ipQc6F9sM3+znYo9fVGZuycxzMvOcddMHD/swSR3VqsxZs2HYh0nqqFZlTgz9MEkd1qbcWctBwz5MUoPq3INnE7AjM3dGxBnAuX3P+dLq+g8An83Me4EdEfHM6v7/RG/3QoBbgSdW1+cfB3A/cFhNfZfUPWaOpCaZOZKaZu5IWlKdAzwfA9ZExI3Ab9DbjRDgQeDJEXEd8G30JvMCeCXw1ojYTm8G9/n73wb8WERcDRzd1/7FwGOdBExSxcyR1CQzR1LTzB1JS1pTV8OZuQd43iKLDh1Qv41vjkL33//PwFl9d/18df/dwJNW3lNJq4GZI6lJZo6kppk7kg6k7kmWJUmSJEmSVLPa9uBpk9yzl9kv3lJb+1Of3TZ07VweaM6zh1p7/HFF9Q+c95ii+uk9c0X1O48t+8gceVXZBGzrLr9x6Nq5KByfjLLyqfVlfZ/bVXa2tqmDCtvfM/zZCorb3u0ZWEYpd+1mbvtN9bW/b+/Qtce8qKwfccghRfVTJ5Rl1Jde+rCi+kf8/b1F9cyUZVr+881D104dcXhZX+57oKg8pqeL6ks+BwBMlbVfova+a0k5N8fc/ffX+ATDb7vMfP0bRU1v+MbJRfVzx5R9D3d/S9l0IlN7yzJk/aaNRfUzX/1aUX2tSjMhy9ZNsdLtuhI5W1/bk6zwd01dbc/edXdR05sLt5Nzrux1RhT+6Chsv9b1rs5zDx5JkiRJkqSO6+wAT0QcHhE/Pu5+SJoMZo6kppk7kppk5kjd19kBHuBwwACS1BQzR1LTzB1JTTJzpI7r8gDPbwCnVqfxe+u4OyNp1TNzJDXN3JHUJDNH6rguT7L8RuDMzNw87o5ImghmjqSmmTuSmmTmSB3X5QGeJUXEBcAFAOvZMObeSFrtzBxJTTJzJDXN3JHar8uHaC0pM7dk5jmZec5ayk6FJ0mlzBxJTTJzJDXN3JHar8sDPPcDh427E5ImhpkjqWnmjqQmmTlSx3V2gCcz7wL+MSKucxIwSXUzcyQ1zdyR1CQzR+q+Ts/Bk5k/MO4+SJocZo6kppk7kppk5kjd1ukBntbIrK3puQcerK3t5Tjk6/uK6m9/2lFF9Ufd+C/DF5eu96npsvqIsvpCc3v2lD2g4PUWt63RCoip4T8/OVdjXwpl4Wfnpp84uqj+Wy6eLaqf/uqdRfV51OFF9SXflZmvfLWo7VJZ9wdhrmzdl8ga29aBxZpppg8/cuj62bvurrEzZX87j/voLUX1t776lKL6kz74paL6285/ZFH9QdfsKqqvU6xdV1SfM2XbdHVu7/baN0c6p+T7XufnpzB3vuvkpxTV33zh2UX1p72/bNtlz/Ebi+qn9pZ9V6Y+d+3wxaV/zwt/X5VsHwPkzExR/dRhZUcYzt1/f1F9F3T2EC1JkiRJkiT1OMAjSZIkSZLUcWMb4ImICyPiORHx4oh404Jlp0fEByJiKiIuW6KNb4mID9ffW0ldZ+ZIapKZI6lp5o6kce7B8xTgcuDZwKULlj2zuu/xwHWLPTgi1mTm1zLzpbX2UtJqYeZIapKZI6lp5o404RqfZLk65d75wMnAZcCpwHOrkeKLgXcAjwRuAw4D5iJia2aeExGvAv4jcCgwHRGvBP46M89s+nVI6gYzR1KTzBxJTTN3JM1rfIAnM18fER8Cfgj4KeCSzHx6X8nmarfBpwHvBd6Wmdf3LT8bOCsz746IkxrqtqSOMnMkNcnMkdQ0c0fSvHEdonU2cA1wBnBj/4KI2ADsycwETgNuWvDYT2TmAc/vGREXRMTWiNi6D08ZLU24ZjMnzRxpwjWaOXvndo+o25I6zN9XkprdgyciNgPvBx4O3Als6N0d24CnAhfRC6XDI2I7cBKwNSIuzMyLqmYeHOa5MnMLsAVgYxyZI3wZkjpibJkzZeZIk2hcmbNp7TFmjjSh/H0lqV+jAzyZuY3eLoKfA55BbxfBt2TmDVXJiyLi9cDNwF3A8zPzDU32UdLqYeZIapKZI6lp5o6kfo0fohURxwA7MnMOOKMvfOY9C/gsvZneP910/yStLmaOpCaZOZKaZu5ImjeOSZbvAF5QXT93keUvrK7+yiLL3k9vF8T527cCzvAuaSAzR1KTzBxJTTN3JM1rfIBnHGJ6iulDNw5dP3vffWXtH3TQ0LW5p2xCsty1q6h+/W1l9dP3ltXf/aSji+pn10dR/fSmgvfpnnuL2mZutqw+O3xocZf7vgrE9Bqmjjpy6PrZ224va3/N8NGds4Wf+0InfLKs/v4TpovqN60fPl8B9h57SFH99I1l/SlSmjnSMuX6dew788Sh66c+fcC5VBuTu8u2i078P18sqp87ZvgsBtj45X1F9Xe+8PSi+qM+dM3QtXOF24CdFwXbjG7njF8EsW7d0OWlv4GKlH4eCreNTv2Tst+G3FGWsXnC8L9/AO591MFF9Udds2Ho2rkHdxa1XfLbrfeAwgOIZmaKyuOIw8va3zd85s+VfobHlFPjOouWJEmSJEmSRsQBHkmSJEmSpI5b0QBPRFwYEc+JiBdHxJsWLDs9Ij4QEVMRcdnKuilJZo6kZpk5kppk5khaqZXuwfMU4HLg2cClC5Y9s7rv8cB1K3weSQIzR1KzzBxJTTJzJK3IsgZ4IuKtEbEdeBJwGfCfgd+PiF+MiGdGxDbgLcBPA38DnB8RW6vHvioi3tnX1l9HxHnV9Qeqtq+PiH+IiCdHxCURcXNEvKjv8X9Z3f8vEfE/V/D6JXWAmSOpSWaOpCaZOZJGZVkDPJn5euCH6Z1S70nA9sw8KzPfnJmfyczNwE3AY4FPAM/LzHOGaPoQ4FOZ+TjgfuBXge8AXgK8ua/uycD3AGcBL4uIYdqW1FFmjqQmmTmSmmTmSBqVlZwm/WzgGuAM4Mb+BRGxAdiTmRkRp9ELpGHsBT5WXb+2amNfRFwLnNRX94nMvKt6rj8DngFsXdCHC4ALANZH2WlzJbVSdzJn6tCClyWppTqTOQcdtKngZUlqqVZnTrXsm9s6DH/qbUnNKR7giYjN9EaXHw7cCWzo3R3bgKcCF9ELpsOrXQ1PArZGxIWZeREww0P3HFrfd31f5r+fMH4O2AOQmXMR0d/XhSeV3+8k85m5BdgCsGnN0eM5Cb2kFetk5qw91syROqqLmbPxsBPMHKmjupI51eO+mTtTR5k7UgsVD/Bk5jZgc0R8jt7o7nuBt2TmDVXJiyLi9cDNwF3A8zPzDX1N3Ar8eERMASfQ2yWw1HdExJHALuDFwGuW0YakDjBzJDXJzJHUJDNH0igtd5LlY4AdmTkHnNEXQPOeBXyW3mzvn16w7B+BW4AbgLcDVy2jC1cAHwG2Ax/JzP12IZS0epg5kppk5khqkpkjaVSWNQdPZt4BvKC6fu4iy19YXf2VRZYl8IoB7R7ad/2XBi0DvpKZLy7uuKROMnMkNcnMkdQkM0fSqKxkkuXuiIB1a8vqC+TevYUdKhBlO1lNbf+XsuaPOrKo/qjPfq2o/uvfdUJR/Te+/7FD1x777s8XtZ0zM0X1czt3FtUXm5ouqy/4WK454fiipme+/JWyvmhpAVGYIyVKPsuxdl1R27G27M/CoX97TVH9IY87tag+H3iwqH76wbLJZnf+h+FPFLLxmtuK2p659ctF9eRcWX2hqQ1lE2LGmuE/C7FpY1HbZs5oxcwca29/YOj62Rr7MnXwwWUPKPzcz95+R1H99N59RfUH79xd1v6eI4rq737ptw5de8QfX1nUds4WvrNZ7xQqcdBBRfXTxxw9dG3uKdv2nr2j7HOjA4vpKaYOG/6kErN79tTWl+ljjimqn7vn3qL6qVvK/mbFkWW5cPD1Xy2rv7Hs92GuX3/gosrsE08ranv62lvL+lL43WVfWYbHrrIML9l+mT667H2d+8ItRfXFGT6gvHMDPJn5fnoTkUlS7cwcSU0ycyQ1ycyRVpdlzcEjSZIkSZKk9iga4ImICyPiORHx4oh404Jlp0fEByJiKiIuW05nqtnjJQkwcyQ1z9yR1CQzR9Iole7B8xTgcuDZwKULlj2zuu/xwHUljUbEGoDMfFphfyStbmaOpKaZO5KaZOZIGpmh5uCJiLcC5wMnA5cBpwLPjYgPAxcD7wAeCdwGHAbMRcTWzDwnIqaB3wDOAw4Cfjcz3xUR59GbCX4HcAbw6Ih4IDMPrZb9EnAncCZwJfCDmZkR8Xzgt4AH6Z0W8JTM/O6VrghJ7WHmSGqauSOpSWaOpDoMNcCTma+PiA8BPwT8FHBJZj69r2Rztdvg04D3Am/LzOurZT8M3JuZT4qIg4B/jIiPV8vOBs7MzMWmmH4C8Djga/SC5ukRsRV4F/CszLwlIv54UJ8j4gLgAoD1U8PP8C5p/DqfOdNmjtQ1Xcudh2TO2rKzmEkav65lDvj7SuqCkkO0zgauoTcafGP/gojYAOzJzAROA27qW/ydwA9FxDbgn4CjqhqAKwaEz/yyr2TmHLANOKl67pv7HjMwgDJzS2aek5nnrJsa/tRwklqjw5lTeJpgSW3Rmdx5SOZMbyh8mZJaojOZA/6+krrggHvwRMRmeqfOezi9Xfo29O6ObcBTgYvoBcPhEbGdXlBsjYgLM/MiIID/lpl/v6Dd8+jtBjjInr7rs8P0VVL3mTmSmmbuSGqSmSOpLgfcgyczt2XmZuALwGOBTwHnZ+bmzNyVmS8C3g38GPA64A+qZRdVTfw98GMRsRYgIh4dEYcss783AadExEnV7Zcvsx1JLWXmSGqauSOpSWaOpLoMdYhWRBwD7Kh25zsjM29YUPIs4LP0Znr/9IJl7wFuAK6KiOvoHeO5rNHizNwF/DjwsYi4ErgfuHc5bUlqLzNHUtPMHUlNMnMk1WHYSZbvAF5QXT93keUvrK7+yiLL5oCfrS79Lqku/bWHVv8+ZFlmvrav7OLMPCMiAvhdYOswr0FSd5g5kppm7khqkpkjqQ5dPO7yv0TEK4F1wNX0RqyXlsDMzNBPMHVw2QSpc7v3HLioEtNR1PbUoWV7W87ee19RPTvuKSqPNWUfmeP/avj1DnDP/xl+wrapf3hkUduzXxw039wAmWX1peZma2t65stfqa3tCVScOTkzy+xdO2rv2FByrqg8Tn5EUf3s9TcduKi//e3/UlSfa8syZ+rGW4vqN9551NC1t799XVHbR71kuqg+99WXCQBzO3fW1/gDS035oGUoy519M/CNO+vrzdTwn+XcV/Z3/8HvPLOo/pCPbS+qjw1l23Q5VbadtvarZdtRs48+bujaKMw/Cvuee8v+PpRuF+We4bePAWa+8tXhiws+kxpK+e+ruSR37a67X0OZvevuovpb/uhxRfWnvLJsW2d6776ieg4q276g4LcnQN5///BduelrZW3Plm27zD1YuC1S+Htp7vbCdV/S/h131df2CHVugCczfxv47XH3Q9JkMHMkNc3ckdQkM0daPUpOky5JkiRJkqQWcoBHkiRJkiSp41o9wBMRPxUR11WX/x4RJ0XEjRHx7oi4PiI+HhFlB1dL0gBmjqSmmTuSmmTmSKtbawd4IuKJwKuBpwDnAv8FOAI4DfjdzHwccA/wPWPrpKRVw8yR1DRzR1KTzBxp9WvzJMvPAP48Mx8EiIg/A54J3JKZ26qaK4GTFntwRFwAXACwfqrsTFSSJtLoMocNtXdW0qqw7Nx56HbOoY10VlLnjW5bJ/x9JbVRa/fgWUL/eeFmGTBIlZlbMvOczDxnnXsZSlq+4sxZG+ub6Zmk1eqAufPQ7RwzR9KKLOP3lbkjtVGbB3g+A7w4IjZExCHAS6r7JKkOZo6kppk7kppk5kirXGsP0crMqyLi/cAV1V3vAXaMr0eSVjMzR1LTzB1JTTJzpNWvtQM8AJn5W8BvLbj7zL7lb2u2R5JWMzNHUtPMHUlNMnOk1a3Nh2hJkiRJkiRpCK3eg2dUcnaW2XvuHXc3AMi5svrZHWV7TcbadUX1uW+mrH52tqienTuLyg/77uFr45iji9pec9zDiupnTjy2qH76zvuL6vc+4oii+nVX/+vQtbe9/LFFbR+95fKiejLL6idNJrlv77h7AUDOlH3HZ6+/qewJIsrqC0NwbteusvYLzd0//Pf22P9yXFnbp59SVL/rpI1F9Yde+/Wi+rnDy860NHX78H9/7jv3xKK2N/z5PxXVa2k5O1u8vVBkbvi//VlQC7Dhb7YduKi//amyzJnbcU9Z+7fXu1101BdvGb54fdkktlMHl9XvecaZBy7qb39vWX6v2VGW3/HV24au3fG804va3vjHNWfOBG4W5dwccw8+OO5u9BTmzim/WVafe8u26eburveot7nC31dFbX9j+O8hwNTjy76Lsa5s+GFu3XRR/QOPKDu50sa/uHro2ukTji9qe+bWLxfVl36OB3EPHkmSJEmSpI5zgEeSJEmSJKnjHOCRJEmSJEnquE4O8ETED0bEFRGxLSLeFRFlB+dJUiFzR1KTzBxJTTJzpNWhcwM8EfEY4OXA0zNzMzALvGK8vZK0mpk7kppk5khqkpkjrR5dPIvWc4EnAp+P3tlbDgZuX1gUERcAFwCsZ0OT/ZO0+hwwd8wcSSNk5khqkr+vpFWiiwM8AXwgM9+0VFFmbgG2AGyMIyfw5IWSRuiAuWPmSBohM0dSk/x9Ja0SnTtEC/gk8NKIOBYgIo6MiBPH3CdJq5u5I6lJZo6kJpk50irRuQGezLwB+Hng4xGxHfgEcPx4eyVpNTN3JDXJzJHUJDNHWj26eIgWmXkRcNG4+yFpcpg7kppk5khqkpkjrQ6d24NHkiRJkiRJD9XJPXg0WO7bO+4uNGbm698oqo81ZR/3v7/yY0X1z3/CdxbVT19ya1H9bEHt0e+6rKhtabmmN20sqs99M2VPsGdPWX2hnBm+P6WZM7XjnqL6u958alH9+t84vKh++uavFdXP7do9dO0hf3VlUdvOzKl5xdstvTP8DN9+dvfTNrd7+O8gQBTkGcBXnrOuqH52fdm6PO2Xv1jW/oM7h67d+Mf/VNQ2Hf4caPTy6uvLHlCYO3M7h/8sL6f9WpV+V275alH53qc8uqj+375rbVH9CZ+eK6qf2jj8duzcbXcUtc1cya+30XEPHkmSJEmSpI4b2wBPRFwYEc+JiBdHxJsWLDs9Ij4QEVMR4a4IkkbC3JHUJDNHUpPMHEnj3IPnKcDlwLOBSxcse2Z13+OB6xrul6TVy9yR1CQzR1KTzBxpwjU+B09EvBU4HzgZuAw4FXhuRHwYuNVL13sAABhESURBVBh4B/BI4DbgMGAuIrZm5jkR8SrgRcCG6nF/nplvaPo1SOoWc0dSk8wcSU0ycyTNa3yAJzNfHxEfAn4I+Cngksx8el/J5mq3wacB7wXelpn9M2FtBp4A7AFuioh3ZOaXG+q+pA4ydyQ1ycyR1CQzR9K8cR2idTZwDXAGcGP/gojYAOzJ3mkPTgNuWvDYT2bmvZm5G7gBOHGxJ4iICyJia0Rs3Ue9Z2GR1Am15o6ZI2kBM0dSk/x9JanZPXgiYjPwfuDhwJ30dgWMiNgGPBW4iF4oHR4R24GTgK0RcWFmXlQ1058mswx4DZm5BdgCsDGO9NyI0oRqKnfMHElg5khqlr+vJPVrdIAnM7fR20Xwc8Az6O0i+JbMvKEqeVFEvB64GbgLeL7HgEpaCXNHUpPMHElNMnMk9Wv8EK2IOAbYkZlzwBl94TPvWcBn6c30/umm+ydp9TF3JDXJzJHUJDNH0rxxTLJ8B/CC6vq5iyx/YXX1VxZZ9n56uyDO3/7uWjopaVUxdyQ1ycyR1CQzR9K8cU2yLEmSJEmSpBFpfA8e1WvqsMOK6nPXrrIniLIxwalDDymqn92xo6i+RM6VzQX3v3ecVPYEe/cVlce6dWXtz84OXTq1aWNZ03fdXdYXHVjE8LXZ3XkKY/36sgccX/bZjDvKPpuxpuzP2sxttw/fduF3Ng4+uKj+6HdvKKrft3GuqH7NxrK/DyVibdl6n73n3pp6os4pyUrKv4e5b6as/cLPMoXbFjlTsK1Qus1V+Lf/YVuH364AmFtT9l5NHb6prP6oI4au3XPS0UVtT196TVE9c2XrZmJ1dVtnarqofPrURU8qNlB+446i+qlDyv7+l5q77/7ha3fuLGy8bFvkoDvLfnseeV1Z5k/vKvvu5u7dw9fOlP09GRf34JEkSZIkSeq4sQ7wRMSFEfGciHhxRLxpwbLTI+IDETEVEZeNq4+SVhdzR1KTzBxJTTJzpMk27j14ngJcDjwbuHTBsmdW9z0euK7hfklavcwdSU0ycyQ1ycyRJthY5uCJiLcC5wMnA5cBpwLPjYgPAxcD7wAeCdwGHAbMRcTWzDwnIqaB3wS+C5gD3p2Z7xjDy5DUIeaOpCaZOZKaZOZIgjEN8GTm6yPiQ8APAT8FXJKZT+8r2VztNvg04L3A2zLz+mrZBcBJwObMnImIIxvsuqSOMnckNcnMkdQkM0cSjPcQrbOBa4AzgBv7F0TEBmBPZiZwGnBT3+JvB96VmTMAmbnoKVYi4oKI2BoRW/exp47+S+qe2nLHzJG0CDNHUpP8fSVNuMb34ImIzcD7gYcDdwIbenfHNuCpwEX0QunwiNhObzR5a0RcmJkXDfs8mbkF2AKwMY5s0Xn5JDWtidwxcyTNM3MkNcnfV5LmNT7Ak5nb6O0i+DngGfR2EXxLZt5QlbwoIl4P3AzcBTw/M9/Q18QngB+JiIvndyEcNMosSWDuSGqWmSOpSWaOpHljOUQrIo4BdmTmHHBGX/jMexbwWXozvX96wbL3AP8GbI+Ia4AfqLu/krrP3JHUJDNHUpPMHEkwvkmW7wBeUF0/d5HlL6yu/soiy2boTRz2U3X2UdLqYu5IapKZI6lJZo4kGO8ky5IkSZIkSRqBsezBoxrt21dWPz1dVj87W1See/eWtd8i73v384vqT5j656L6qUd8S1E9t981dOmXfuSMoqYf/uufK+uLDiwnY+7B3L277AFfL6ufe+DBovpYU/hnreB9isK8zJmZovoNt95TVM9MWR5/4zuOL6o/9rL1wxfPFTUN99xb+ACpJ/eVfa9KRURRfZb+V2nR34bSL1aZQ28sm2IlD15XVD977Kay+oPXDl37wAllfSnriYbW1W2dLPxu3XZnUfncgzuL6nN34RnJCvufhb/fihS2HXvKfqse84kvFdXv/WDhb9sXDb8uS7cxc894zjTnHjySJEmSJEkd1/gAT0RcGBHPiYgXR8SbFiw7PSI+EBFTEXFZ3/3nRcRfN91XSd1n5khqmrkjqUlmjqR549iD5ynA5cCzgUsXLHtmdd/jgesa7pek1cnMkdQ0c0dSk8wcSUCDc/BExFuB84GTgcuAU4HnRsSHgYuBdwCPBG4DDgPmImJrZp6zoJ0jgfcCpwA7gQsyc3tTr0NSN5g5kppm7khqkpkjaaHG9uDJzNcDPwy8H3gSsD0zz8rMN2fmZzJzM3AT8FjgE8DzFoZP5ZeBqzPzLOBngQ828gIkdYqZI6lp5o6kJpk5khZq+hCts4FrgDOAG/sXRMQGYE9mJnAavTBazDOA/wuQmZ8CjoqIjQuLIuKCiNgaEVv3MZ4ZrCWNnZkjqWmN5I6ZI6nito6kf9fIIVoRsZneyPLDgTuBDb27YxvwVOAieqF0eERsB04CtkbEhZl50XKeMzO3AFsANsaRHT2Hn6TlMHMkNa3p3DFzpMnmto6kxTSyB09mbqt2EfwCvV0EPwWcn5mbM3NXZr4IeDfwY8DrgD+oli0WPp8BXgG92d+BOzPzviZeh6RuMHMkNc3ckdQkM0fSYho7RCsijgF2ZOYccEZm3rCg5FnAZ+nN9P7pJZr6JeCJ1Uj0bwCvrKG7kjrOzJHUNHNHUpPMHEkLNXYWrcy8A3hBdf3cRZa/sLr6K4ssuwS4pLp+N/DiuvopaXUwcyQ1zdyR1CQzR9JCjQ3wqBlzu3fX+wQRZfV7WjQB29xsUfnx77yiqD7XrSurf+DBovqpwzcNXfvHF/xWUduv//X9tgmWVvo58CjtVWv2nnvLHlD82Sn78ORs2fe8qO29e4vqS76zADk9XVQ/c/ShRfUP++Q3iuo5aPhM+/Lzjypq+luur/dzoA4pfW+zvu84wNzuwvZLM61E4bqZK9yuiF1l24xx8Pqy+rmy/u990qlD1+540c6ito/4i7K+z+0sa9/tnI4p/G7N3lfv0WpZ+Bul075+e1H5DReeXlR/4tvK3tu1xxcMhxRsFwHEF24pqs99ZduZgzR9Fi1JkiRJkiSNmAM8kiRJkiRJHTeyAZ6IuDAinhMRL46INy1YdnpEfCAipiLishE+56si4p2jak9Sd5g5kppm7khqkpkjqdQo9+B5CnA58Gzg0gXLnlnd93jguhE+p6TJZeZIapq5I6lJZo6kIiueZDki3gqcD5wMXAacCjw3Ij4MXAy8A3gkcBtwGDAXEVsz85yIeBXwEmATcALwh5n5y1W7Pwi8DlgH/BPw45k5GxGvBt4E3ANcA7RoFl9JdTNz9P/bu/tgO+r6juPvb0JCEiGhgWBhEOPwIKClqaaAYEBarC2iQrFDa6dOmGkz2iozWEBqaacFNB10dLT0YdKWB8c+xCFoOy1VGSRSClpSEiMQSQWp8iAESAiBJOTe++0f50RvLzfk/nLP7tm95/2aucOe3e/97XcP537mnF/27Ep1M3ck1cnMkbSvJj3Bk5mXRsQXgfcDHwFWZ+Zpo0oWdU8bPBW4DvhUZt4/avtJwBuBF4F7IuLfgBeAC4DTMnNXRPwl8JsRcSvwp8CbgefoBNza8fqKiGXAMoBZzJnsYUpqCDNHUt2amDtmjjR1NTFzwNyR2qBXt0l/E53Z3uOADaM3RMQcYGdmZkQcAzw45ndvzcxnurU3A28FhuiEzD3Ruf3kbOApOqcprs7MTd36lcCx4zWUmSuAFQBzY743L5SmFjNHUt0alTtmjjTlNSpzwNyR2mBSEzwRsQi4ATgCeBqY01kd64C3ACvphNJBEbEeWAisiYjlmbmyO8zYcEgggBszc+zFxM6dTL+S2s3MkVQ3c0dSncwcSZMxqYssZ+a6zFwEbAROAL4OvCMzF2Xm9sx8N/A3wAfpfN/zr7vbVo4a5u0RMT8iZgPnAv8J3Aa8NyIOBehufy2d74qeEREHR8QM4Ncm07+kdjFzJNXN3JFUJzNH0mT04iLLC4DNmTkSEcdl5gNjSk4HPk/n+5rfGGeI/wJW0Zml/kJmrumOewXwtYiYBuwCfi8zvxkRf0LnYmNbgHWT7V9Su5g5kupm7kiqk5kjaV/14iLLm4B3dpdPGWf7u7qLV+1hiEcz82WnBnZnoVeOs/564Pp9blhSq5k5kupm7kiqk5kjaV/16iLLqkrnImjVDT99elF9jpRdTy32K3uJTTvwwAnXDj+3tayXwmOdfsj8ovqRwn6mH/7TRfW5ecuEa3//obKza6fxw6J60uvqTVmlmVP6Wqj6tVM6fsHxTj/i8LJWtjxX1sv3Hikq32/GjKL64eMXlo3/xOYJ1247al7R2DFzZlF97hoqqme4rFx9VHXmVPw+iii82kFBOzGtrPfi91w/dVBRfe7YUVTPzLJ+Zj38zMR7yYm/XwRguDAUSv+/qlWmH1z2Hn/4mWeL6kv/Fks/XzFS+nqeeJaUfl6K2bOL6kdefLGo/oRrniyq3/G6Q4rqef6FCZc++p7DioY+oqwTePypsvo9vCz7OsGTmTfQuYiYJFXOzJFUN3NHUp3MHGmw1T49HRHLI+LMiDg3IsZexf31EXFjREyLiLtHrV8cEZ+ru1dJ7WfmSKqTmSOpbuaOpN36cf7hycA3gTOAO8ZsW9Jd9zPAfbtXZuaazLyotg4lTSVmjqQ6mTmS6mbuSAJq/IpWRHwSeAfwOjpXaT8K+MWIuAm4Hfhz4EjgSeBAYCQi1mTm4oh4G3BJZp4TEScBnwVmAduBCzPzwbqOQ1I7mDmS6mTmSKqbuSNprNomeDLz0oj4IvB+4CPA6sw8bVTJou5pg6cC1wGfysz7xxnqu8CSzByKiLOATwDnV9y+pJYxcyTVycyRVDdzR9JYdV9k+U3At4HjgA2jN0TEHGBnZmZEHAPsadZ4HnBjtyaBcW8jEhHLgGUAs5jTm+4ltY2ZI6lOZo6kupk7kn6slgmeiFhE52ruRwBPA3M6q2Md8BZgJZ1QOigi1gMLgTURsTwzV44Z7irg9sw8LyIWAqvH22dmrgBWAMyN+d7TWRogZo6kOpk5kupm7kgaTy0TPJm5js4pgncBb6VziuA1mflAt+TdEXEp8DDwDHB2Zl62h+HmAY91l5dW17WktjJzJNXJzJFUN3NH0nhqu4tWRCwANmfmCHDcqPDZ7XTgTjpXev/GKwx1DbA8ItZS/1fMJLWEmSOpTmaOpLqZO5LGqvMiy5uAd3aXTxln+7u6i1eNs2013VMFM/Nu4NhRm6/ocauSpgAzR1KdzBxJdTN3JI1V2xk8kiRJkiRJqoan4DVdVnv9shwaKvuFiKLykZ07y8Z/aVdZfYkcKSof2bylqD4OeFVRfW59vmz8WbMmXHvywY8Ujb1m/8I7IYwUvi5fKitXH1WcOU0T06dPuDZnjntTkT0a2b6jqD5fKvtDmTajrJ9p9z9cVM+rF0y49FXfL3s7MXTqG4rqZz5alsdsLCtXH1WdOZWPP1xWP23imUOU/TtsFr7nGnrs8aL62K8sc6IwM/OIiWfOri0Tf08EMHTy8UX1M9Y/UlTPs2Xl6q/hZzdXOn7x56uqVZiDI9teKKrPobLPesOP/6iofv9tLxbVj2zdOuHa19z0w6Kx5/3DtqL6LRfOL6rfU+54Bo8kSZIkSVLLOcEjSZIkSZLUck7wSJIkSZIktZwTPJIkSZIkSS3nBI8kSZIkSVLLTdm7aEXEMmAZwCwK7xAkSYXMHEl1MnMk1c3ckZpvyp7Bk5krMnNxZi6ewf79bkfSFGfmSKqTmSOpbuaO1HxTdoJHkiRJkiRpULR+gicibomIw/vdh6TBYOZIqpOZI6lu5o7UXq2/Bk9mnt3vHiQNDjNHUp3MHEl1M3ek9mr9GTySJEmSJEmDrvVn8KhmmRWPP1zd0COF9UNDRfVRWF88/oyZE65d8/Ozi8bmxKOLyj+8clVR/a1HFZVLtSn5Oxze+FCFnZTL7dvL6ofL8jUfe2LCtUf+3fNFYz+/pCxzXrgsiur55bJyqTYjE/87zILafTFt1qyy+sNeXVQ/9MgPiurjvu9NuPboLxxfNPaMJ7cW1W+8dmFRPe8rK1efVf15psVKP59ULWZO/PMPwPCmTWU7mDZ9wqUjT5aNveljZTn1SzffUVT/1TeMv94zeCRJkiRJklrOCR5JkiRJkqSWq2SCJyIWRsT2iFjXfTwcEetG/VzeXb86ItaM+r3FEbG6u/y2iHguItZGxIMRcUdEnDOq9uKI+EFEXFvFMUhqF3NHUp3MHEl1MnMkTUSV1+B5KDMXdZe3j1oe69CI+JXM/Pdxtv1HZp4DEBGLgC9HxPbMvC0zPxMRm4HFFfQuqZ3MHUl1MnMk1cnMkfSKmvAVrU8Cf7i3osxcB1wJfKjyjiRNdeaOpDqZOZLqZOZIA6quCZ7ZY04hvGDUtruBlyLizAmMcy9w3ER2GBHLImJNRKzZxc596VlSu9WaO2aONPDMHEl18vOVpJep6zbpr3QKIcDVwBXAR/cyzoTvk5qZK4AVAHNjvvfCkwZPrblj5kgDz8yRVCc/X0l6mSZ8RYvM/DowGzhlL6U/B2yoviNJU525I6lOZo6kOpk50mBqxARP19XAZXvaGBEnAn8E/EVtHUma6swdSXUycyTVycyRBkxdX9GavfuWfl1fyczLRxdk5i0RsWnM7y2JiLXAHOAp4KLMvK3iXiVNDeaOpDqZOZLqZOZIeplaJngyc/oe1r9tzOM3j1peDcyrtDFJU5a5I6lOZo6kOpk5ksYTmb2/PlZEvAa4C3hmLxf/msw+LgY+AKzKzI/tpXYT8L/jbDoEeLpgtyX1VY7d9vom9dK0+ib10sv612bmgoJxijUpd/qUOVXXN6mXptU3qZem1Zs5k9vHIGdO1fVN6qVp9U3qpWn1Zs5PanuRO036f9u0+ib10vb6JvXSy/rxcyczB/YHWFNVfZVjt72+Sb00rb5JvdRRP2g/TXv+fW363AzaczNoP017/ptU36RemlbfpF6aVm/m9Pf5HKT6JvXS9vom9VJHfZMusixJkiRJkqR94ASPJEmSJElSyw36BM+KCuurHLs19RGxbczjpcDzPRp7dUQsHmfTSxHx/YhY1/3Z2/eUm/RcNqmXOuoHTdOe/yn52uxD7qyIjo9HxMaI2BARF5WOPUXrm9TLIGra89+k+jZnDsC8Ue9zHo+IL+/L+D2oHbR6M2fvfK31pr5pmXNHRNzbzZw7I+LofRm/T/VN6qXy+kousiztFhHbMvOAUY+XAosz80M9GHs1cElmrhmz/gbgXzPzpsnuQ1L79Cl3LgTOBJZm5khEHJqZT012f5Karx+ZM6ZmFfDPmfn5ye5PUvP16X3ORuA9mbkhIn4XOCkzl052f+q9QT+DR30UEQsiYlVE3NP9Oa27/qSIuDsi1kbEXRHx+u762RHxT91/Hf8SMLuvByCpdSrMnQ8CV2bmCICTO5Kg+vc6ETEX+AVgb2fwSBoAFWZOAnO7y/OAxys/GO2T/frdgKa82RGxbtTj+cC/dJc/C3wmM++MiCOBrwLHA98FlmTmUEScBXwCOJ/OB6gXM/P4iDgRuPcV9vvxiPhj4Dbg8szc2dvDktRg/cido4ALIuI8YBNwUWb+T8+PTFIT9eu9DsC5wG2ZubWHxyOp2fqROb8N3BIR24GtwCk9Pyr1hBM8qtr2zPzxNXB2n0LYfXgWcEJE7N48NyIOoDMrfGNEHENntnhGd/vpwOcAMnN9RKzfwz7/APgRMJPOdxY/ClzZqwOS1Hj9yJ39gR2ZuTgifhW4DljSu0OS1GD9yJzdfgP4214chKTW6EfmXAycnZnfiohLgU/TmfRRwzjBo36aBpySmTtGr4yIa4HbM/O8iFgIrC4ZNDOf6C7ujIjrgUsm36qkKaKS3AEeBW7uLn8JuH5ybUqaIqrKHCLiEOAk4LzJtylpiuh55kTEAuBnM/Nb3VUrga/0pFv1nNfgUT99Dfjw7gfxk7tdzQMe6y4vHVV/B/C+bu0bgRPHGzQiDuv+N+icunxfL5uW1GqV5A6d61+c2V0+A9jYm3YltVxVmQPwXjo3ldjxCjWSBksVmbOZzp37ju0+fjuwoXctq5ec4FE/XQQsjoj1EfEA8IHu+muA5RGxlv9/ltlfAQdExAY6X7n67z2M+/cR8R3gO8AhwNWVdC+pjarKnT8Dzu9mz3I8bVlSR1WZA/DrwD9W0LOk9up55mTmEPA7wKqI+DbwW8ClFR6DJsHbpEuSJEmSJLWcZ/BIkiRJkiS1nBM8kiRJkiRJLecEjyRJkiRJUss5wSNJkiRJktRyTvBIkiRJkiS1nBM8kiRJkiRJLecEjyRJkiRJUss5wSNJkiRJktRy/wdUAjXaiBQquwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1152x576 with 8 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "opsiugQbC51o"
      },
      "source": [
        "### Exporting Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Go4L6nwJDAHd"
      },
      "source": [
        "Define translater exporter:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yDO3IGygC7-p"
      },
      "source": [
        "class ExportTranslator(tf.Module):\n",
        "  def __init__(self, translator):\n",
        "    self.translator = translator\n",
        "\n",
        "  @tf.function(input_signature=[tf.TensorSpec(shape=[], dtype=tf.string)])\n",
        "  def __call__(self, sentence):\n",
        "    (result, \n",
        "     tokens,\n",
        "     attention_weights) = self.translator(sentence, max_length=100)\n",
        "\n",
        "    return result"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jC4OA8sxDDbQ"
      },
      "source": [
        "export the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ol6sZIeGDFcd",
        "outputId": "51367822-d3ec-4486-a04f-4b6b977ae20a"
      },
      "source": [
        "translator = ExportTranslator(translator)\n",
        "tf.saved_model.save(translator, export_dir='translator')"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Found untraced functions such as embedding_layer_call_and_return_conditional_losses, embedding_layer_call_fn, dropout_8_layer_call_and_return_conditional_losses, dropout_8_layer_call_fn, embedding_1_layer_call_and_return_conditional_losses while saving (showing 5 of 560). These functions will not be directly callable after loading.\n"
          ]
        }
      ]
    }
  ]
}