projet_WDS.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "colab": {
      "provenance": []
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6ff2cfe533b94ec19835478e945c9b46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f9f8dec224c49f8a8ac78dc8f829e43",
              "IPY_MODEL_7f29a31c548b4d96a2a9540e86ef287c",
              "IPY_MODEL_60c02f192ada47cda73bd7a4be157721"
            ],
            "layout": "IPY_MODEL_cd2d79bd25e54f8c972a75a7978298d3"
          }
        },
        "4f9f8dec224c49f8a8ac78dc8f829e43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_742ecfbb7baf4a6ea90267cc7147d243",
            "placeholder": "​",
            "style": "IPY_MODEL_9f03a8a2db4e4a5b9d525059926e5807",
            "value": "Downloading: 100%"
          }
        },
        "7f29a31c548b4d96a2a9540e86ef287c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bebc6c90a5c24dd481f81637600589e8",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_90e3bfbb8c424fe2a303443bc340729e",
            "value": 28
          }
        },
        "60c02f192ada47cda73bd7a4be157721": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fccc64ad24946448f653e54eeb44f2c",
            "placeholder": "​",
            "style": "IPY_MODEL_d1f1c466e239451292626a619a0c0e3d",
            "value": " 28.0/28.0 [00:00&lt;00:00, 364B/s]"
          }
        },
        "cd2d79bd25e54f8c972a75a7978298d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "742ecfbb7baf4a6ea90267cc7147d243": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f03a8a2db4e4a5b9d525059926e5807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bebc6c90a5c24dd481f81637600589e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90e3bfbb8c424fe2a303443bc340729e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4fccc64ad24946448f653e54eeb44f2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1f1c466e239451292626a619a0c0e3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ac3180b9f6d244949984240a3451ab60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa23ceeab7274e01a3f5143a7da3c912",
              "IPY_MODEL_12bef55836424effb836e42de88a93b7",
              "IPY_MODEL_244040fdba2b4535b5db6c39d701006d"
            ],
            "layout": "IPY_MODEL_497813ee61e448358edb1626836f8c02"
          }
        },
        "aa23ceeab7274e01a3f5143a7da3c912": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1bd4c49ca3cd4dadaa0600e881d6d593",
            "placeholder": "​",
            "style": "IPY_MODEL_1bda3b1158084d9fb576606e7e0f34de",
            "value": "Downloading: 100%"
          }
        },
        "12bef55836424effb836e42de88a93b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7152dca9bf204b8e93ca44769bfc5e07",
            "max": 1496,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a0ecbeee6ad7450cabb1f50a77dd1701",
            "value": 1496
          }
        },
        "244040fdba2b4535b5db6c39d701006d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0579c7432e34b7586437d41c5eb793f",
            "placeholder": "​",
            "style": "IPY_MODEL_88902c77a50c4362911498301946fbff",
            "value": " 1.50k/1.50k [00:00&lt;00:00, 23.7kB/s]"
          }
        },
        "497813ee61e448358edb1626836f8c02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bd4c49ca3cd4dadaa0600e881d6d593": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bda3b1158084d9fb576606e7e0f34de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7152dca9bf204b8e93ca44769bfc5e07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0ecbeee6ad7450cabb1f50a77dd1701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0579c7432e34b7586437d41c5eb793f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88902c77a50c4362911498301946fbff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e4d45bd05f6f44abb5118585df3e9ac4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d63b066a7d1a4238bb7f1caa64b481f1",
              "IPY_MODEL_9991b634e65b4393b8c692f82a7f5ed7",
              "IPY_MODEL_2b615107feb2419abb52d22955443449"
            ],
            "layout": "IPY_MODEL_0272af9a301e4869a6503af97431a65a"
          }
        },
        "d63b066a7d1a4238bb7f1caa64b481f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dde8fee0f7ec413fb1e8d30b1fb0beb4",
            "placeholder": "​",
            "style": "IPY_MODEL_82a77714a2af4232b4cba8869f0409f3",
            "value": "Downloading: 100%"
          }
        },
        "9991b634e65b4393b8c692f82a7f5ed7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5cc4c71304294efb9c1fe22b7c0cc755",
            "max": 1561415,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_773fa3f733304f888d3bd0d1aa84e385",
            "value": 1561415
          }
        },
        "2b615107feb2419abb52d22955443449": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_011e44fc801c4ee3b7c398899f9cdabf",
            "placeholder": "​",
            "style": "IPY_MODEL_e7d402755f6848b1b81d31a5b8ab5595",
            "value": " 1.56M/1.56M [00:00&lt;00:00, 3.70MB/s]"
          }
        },
        "0272af9a301e4869a6503af97431a65a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dde8fee0f7ec413fb1e8d30b1fb0beb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82a77714a2af4232b4cba8869f0409f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5cc4c71304294efb9c1fe22b7c0cc755": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "773fa3f733304f888d3bd0d1aa84e385": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "011e44fc801c4ee3b7c398899f9cdabf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7d402755f6848b1b81d31a5b8ab5595": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73a6436d36334ad2a6c774efada1647c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c13a831a8f9644cab1b21ea80b6637d8",
              "IPY_MODEL_b7b7bc70dab64924a7c5765130154c1b",
              "IPY_MODEL_381d24095a614ffa9e45c08fa11c0fd4"
            ],
            "layout": "IPY_MODEL_b39bb0936c844685a54e7f2da3a90aa0"
          }
        },
        "c13a831a8f9644cab1b21ea80b6637d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51d5fe625fd44591af86b7c988168004",
            "placeholder": "​",
            "style": "IPY_MODEL_cf9fe539b6fc44078f7b3159e1c9decd",
            "value": "Downloading: 100%"
          }
        },
        "b7b7bc70dab64924a7c5765130154c1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce3dcf6f382f4125af6002a88f4aa277",
            "max": 895731,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e6087e53d7e34bf5ae333fdd5cb910c4",
            "value": 895731
          }
        },
        "381d24095a614ffa9e45c08fa11c0fd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfed4c93d7114446b3a63a967db64809",
            "placeholder": "​",
            "style": "IPY_MODEL_112db6d4bbc34d0aaa66b1b7fa7ab52c",
            "value": " 896k/896k [00:00&lt;00:00, 12.1MB/s]"
          }
        },
        "b39bb0936c844685a54e7f2da3a90aa0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51d5fe625fd44591af86b7c988168004": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf9fe539b6fc44078f7b3159e1c9decd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce3dcf6f382f4125af6002a88f4aa277": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6087e53d7e34bf5ae333fdd5cb910c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dfed4c93d7114446b3a63a967db64809": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "112db6d4bbc34d0aaa66b1b7fa7ab52c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23b53112c64049559dd7df682a087a07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d77bb697c63b4ea68207b2ddeac7a8ba",
              "IPY_MODEL_1dc3b755a6f54a569252e71d897e6670",
              "IPY_MODEL_651b2405314b43528851c65929613042"
            ],
            "layout": "IPY_MODEL_bbb17e9bd31c4b92bce16c87ebcd8874"
          }
        },
        "d77bb697c63b4ea68207b2ddeac7a8ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba5716e4dc774b79a87aaa8797e823c1",
            "placeholder": "​",
            "style": "IPY_MODEL_0d2f9fe61e6d43ceac986353d6504998",
            "value": "Downloading: 100%"
          }
        },
        "1dc3b755a6f54a569252e71d897e6670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7cbd46cff2a440ffa108be0578cb1984",
            "max": 553238687,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_866aaccdb00546d382e3c3c3b50c07d3",
            "value": 553238687
          }
        },
        "651b2405314b43528851c65929613042": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac78666b433b428ebc439554c2f4e5b1",
            "placeholder": "​",
            "style": "IPY_MODEL_3f891515da404620b25a264329861323",
            "value": " 553M/553M [00:09&lt;00:00, 51.5MB/s]"
          }
        },
        "bbb17e9bd31c4b92bce16c87ebcd8874": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba5716e4dc774b79a87aaa8797e823c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0d2f9fe61e6d43ceac986353d6504998": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7cbd46cff2a440ffa108be0578cb1984": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "866aaccdb00546d382e3c3c3b50c07d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ac78666b433b428ebc439554c2f4e5b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f891515da404620b25a264329861323": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LO5XnV__cy3D",
        "outputId": "3966e714-3a82-4c68-8bc5-eacfa647c9b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd drive/MyDrive/'Colab Notebooks'/nlp_projet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DbcwJtgcc77D",
        "outputId": "9f8142a7-7cf2-4b25-c942-341a81561c0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/Colab Notebooks/nlp_projet\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ADMDWc5LgpC0"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "# WSD par fine-tuning d'un modèle *BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sUrBv8Gvg2bE"
      },
      "source": [
        "On utilise ici les données du French FrameNet \"ASFALDA\": dans ces données, certains mots ont été associés à un frame FrameNet\n",
        "- ces mots sont les \"targets\"\n",
        "- **associer le bon frame à un target correspond à une tâche de WSD**\n",
        "- modulo le fait qu'un même frame regroupe plusieurs entrées lexicales (par exemple FR_Commerce_buy => acheter.v, achat.n, acquérir.v, etc...)\n",
        "- dans les données, les phrases contenant plusieurs targets ont été dupliquées: on a une ligne par couple phrase + target\n",
        "\n",
        "Les données FrameNet comprennent également l'annotation des arguments sémantiques, et leur typage au moyen d'un rôle (Buyer, Seller, Goods ...), que l'on ignorera ici.\n",
        "\n",
        "On va construire un classifieur :\n",
        "- entrée = un target et sa phrase de contexte\n",
        "- sortie = une distribution de probas sur les différents sens\n",
        "  - ici les sens sont des frames\n",
        "  - on peut ou pas contraindre que les sens \"permis\" pour un target soient uniquement ceux vus à l'entraîînement pour ce target (pour ce lemme)\n",
        "\n",
        "On utilisera un modèle *BERT pour obtenir une représentation contextuelle du mot target.\n",
        "\n",
        "Mais BERT donne des vecteurs contextuels pour chaque **token**, un token pouvant être un sous-mot.\n",
        "**Dans la version de base, vous utiliserez le vecteur *BERT du premier token du mot target.**\n",
        "\n",
        "Ainsi pour le target *comprenions* dans:\n",
        "\n",
        "*Nous comprenions bien le cours*\n",
        "\n",
        "tokenisé en :\n",
        "\n",
        "'\\<s>', 'Nous\\</w>', 'compren', 'ions\\</w>', 'bien\\</w>', 'le\\</w>', 'cours\\</w>', '.\\</w>, '\\</s>'\n",
        "\n",
        "vous utiliserez le vecteur caché du sous-mot \"compren\".\n",
        "\n",
        "Le classifieur dans la version de base sera un réseau de neurones constitué\n",
        "- d'un réseau *BERT\n",
        "- dont on récupère le vecteur caché du 1er sous-mot du target\n",
        "- et une couche linéaire + softmax sur les différents frames présents dans les données d'entraînement.\n",
        "\n",
        "Le classifieur est unique pour tous les lemmes, et peut prédire tout sens(frame) pour tout lemme target, même s'il s'agit d'un sens non vu pour ce lemme dans les données d'apprentissage.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xp3IN9YQexxx"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "#from tqdm import tqdm  \n",
        "from tqdm.notebook import tqdm # for progress bars in notebooks\n",
        "from random import shuffle\n",
        "import os\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f_c3C5Pzexx-"
      },
      "source": [
        "## Conventions de nommage des variables\n",
        "\n",
        "- on considère des phrases déjà segmentées en mots (avec segmenteur par règles)\n",
        "- (mais pas encore segmentées en sous-mots)\n",
        "- on utilise \"word\" ou \"w\" pour un mot (ou ponctuation)\n",
        "- et \"token\" après tokenisation de type *BERT (BPE ou WordPiece etc...)\n",
        "\n",
        "- on distingue dans les noms de variables \n",
        " - les identifiants entiers des symboles \n",
        "   (pour le vocabulaire des tokens, le vocabulaire des labels ...)\n",
        " - versus le rang d'un élément (token ou mot) dans une séquence\n",
        "- tid => id de token\n",
        "- trk / wrk => rang de token / rang de mot dans une séquence\n",
        "- tg => \"target\", donc \n",
        " - tg_wrk = le rang dans la phrase du mot target\n",
        " - tg_trk = le rang dans la tokenisation *BERT du premier token du mot target\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vfm0CXb9exyF"
      },
      "source": [
        "## Les données \"ASFALDA\"\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n",
        "Il s'agit des données d'un FrameNet du français, comprenant environ 16000 annotations, pour environ 100 frames distincts.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kySX0ye3jdpP"
      },
      "source": [
        "### Récupération des données\n",
        "\n",
        "> Indented block\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D_2cjFvYfOn3"
      },
      "source": [
        "### Lecture des données"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYQhZaBzexyF"
      },
      "source": [
        "# lecture des données\n",
        "\n",
        "def load_asfalda_data(gold_data_file, split_info_file, val_proportion=None):\n",
        "    \"\"\"\n",
        "        Inputs: - asfalda gold data file\n",
        "                - file with list of sentid / corpus type pairs (corpus types train/dev/test)\n",
        "                - val_proportion : if set to value > 0 (and <1)\n",
        "                  the training file is split into train/validation\n",
        "                  so that the validation part represents the provided proportion \n",
        "                  of the original training file\n",
        "        Returns 3 dictionaries (whose keys are corpus types (train/dev/test/val))\n",
        "        - sentences\n",
        "        - rank of target word in each sentence\n",
        "        - gold labels\n",
        "\n",
        "        Example:\n",
        "        sentences['train'] = [['Le', 'code', 'comprend', 'des', 'erreurs','.'],\n",
        "                              ['Comprends', '-tu', '?']]\n",
        "         # the targets are are the 3rd and first words                     \n",
        "        tg_wrks['train'] = [2, 0]\n",
        "        tg_lemmas['train'] = ['comprendre', 'comprendre']\n",
        "        labels = ['frame1', 'frame2']\n",
        "                                \n",
        "    \"\"\"\n",
        "    # chargement de la répartition usuelle des phrases en train / dev / test\n",
        "    s = open(split_info_file)\n",
        "    lines = [ l[:-1].split('\\t') for l in s.readlines() ]\n",
        "    split_info_dic = { line[0]:line[1] for line in lines }\n",
        "\n",
        "    # les phrases de dev / train / test\n",
        "    sentences = {'dev':[], 'train':[], 'test':[]}\n",
        "    # les word rank (wrk) des mots étiquetés en frames (les \"targets\" ou \"tg\")\n",
        "    tg_wrks = {'dev':[], 'train':[], 'test':[]}\n",
        "    # les lemmes des targets\n",
        "    tg_lemmas = {'dev':[], 'train':[], 'test':[]}\n",
        "    # les sens (= des frames) étiquetés pour ces mots\n",
        "    labels = {'dev':[], 'train':[], 'test':[]}\n",
        "\n",
        "    max_sent_len = {'dev':0, 'train':0, 'test':0}\n",
        "    max_tg_wrk = {'dev':0, 'train':0, 'test':0}\n",
        "\n",
        "    stream = open(gold_data_file)\n",
        "    for line in stream.readlines():\n",
        "        if line.startswith('#'):\n",
        "            continue\n",
        "        line = line.strip()\n",
        "        (sentid, tg_wrk, frame_name, tg_lemma, tg_pos, rest) = line.split('\\t',5)\n",
        "        # on ignore pour l'instant l'annotation en rôles\n",
        "        # les phrases sont pré-segmentées en mots (séparateur = espace) \n",
        "        # => on splitte, de manière à utiliser infra le tokenizer en mode is_split_into_words=True\n",
        "        sentence = rest.split(\"\\t\")[-1].split(' ')\n",
        "        part = split_info_dic[sentid]\n",
        "        tg_wrk = int(tg_wrk)\n",
        "\n",
        "        l = len(sentence)\n",
        "        sentences[part].append(sentence)\n",
        "        labels[part].append(frame_name)\n",
        "        tg_wrks[part].append(tg_wrk)\n",
        "        tg_lemmas[part].append(tg_lemma)\n",
        "        if max_sent_len[part] < l: \n",
        "            max_sent_len[part] = l \n",
        "        if max_tg_wrk[part] < tg_wrk: \n",
        "            max_tg_wrk[part] = tg_wrk \n",
        "    print(\"Longueur max des phrases:\", max_sent_len)\n",
        "    print(\"Rang max du target (en mots):\", max_tg_wrk)\n",
        "    \n",
        "    # decoupage du train en train + validation\n",
        "    # (pour réglage du nombre d'époques)\n",
        "    if val_proportion:\n",
        "        # le split sera le même pour les 3 listes\n",
        "        for dic in [sentences, tg_wrks, labels, tg_lemmas]:\n",
        "            (dic['val'], dic['train']) = split_list(dic['train'], proportion=val_proportion)\n",
        "    return sentences, tg_wrks, tg_lemmas, labels\n",
        "\n",
        "def split_list(inlist, proportion=0.1, shuffle=False):\n",
        "     \"\"\" partitions the input list of items (of any kind) into 2 lists, \n",
        "     the first one representing @proportion of the whole \n",
        "     \n",
        "     If shuffle is not set, the partition takes one item every xxx items\n",
        "     otherwise, the split is random\"\"\"\n",
        "     n = len(inlist)\n",
        "     size1 = int(n * proportion)\n",
        "     if not(size1):\n",
        "          size1 = 1\n",
        "     print(\"SPLIT %d items into %d and %d\" % (n, n-size1, size1))\n",
        "     # if shuffle : simply shuffle and return slices\n",
        "     if shuffle:\n",
        "          # shuffle inlist (without changing the original external list\n",
        "          # use of random.sample instead of random.shuffle\n",
        "          inlist = sample(inlist, n)\n",
        "          return (inlist[:size1], inlist[size1:])\n",
        "     # otherwise, return validation set as one out of xxx items\n",
        "     else:\n",
        "          divisor = int(n / size1)\n",
        "          l1 = []\n",
        "          l2 = []\n",
        "          for (i,x) in enumerate(inlist):\n",
        "               if i % divisor or len(l1) >= size1:\n",
        "                    l2.append(x)\n",
        "               else:\n",
        "                    l1.append(x)\n",
        "          return (l1,l2)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "sWMrdBKb_WEk"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QaKnFVJ0exyL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f60fff9-1941-414f-a95a-10f4831da369"
      },
      "source": [
        "gold_data_file = './asfalda_data_for_wsd/sequoiaftb.asfalda_1_3.gold.uniq.nofullant.txt'\n",
        "# les informations pour le split train / dev / test\n",
        "# tel qu'utilisé généralement pour ce corpus\n",
        "split_info_file = './asfalda_data_for_wsd/sequoiaftb_split_info'\n",
        "\n",
        "sentences, tg_wrks, tg_lemmas, label_strs = load_asfalda_data(gold_data_file,\n",
        "                                                              split_info_file, \n",
        "                                                              val_proportion=0.1)\n",
        "\n",
        "# récupération de tous les labels (= les frames) rencontrés\n",
        "all_labels_strs = []\n",
        "all_lemma_strs = []\n",
        "for p in sentences.keys():\n",
        "    all_labels_strs += label_strs[p]\n",
        "    all_lemma_strs += tg_lemmas[p]\n",
        "    avgl = sum([len(s) for s in sentences[p]])/len(sentences[p])\n",
        "    print(\"%s : %d sentences, average lentgh=%3.2f\" \n",
        "          %(p, len(sentences[p]), avgl))\n",
        "\n",
        "#@@ ATTENTION: ici vous codez tous les lemmes, y compris les lemmes du dev / test inconnus du train\n",
        "#   => cela donne une surestimation des performances utilisant les lemmes\n",
        "i2lemma = list(set(all_lemma_strs))\n",
        "lemma2i = {x:i for i,x in enumerate(i2lemma)}\n",
        "\n",
        "\n",
        "# id des labels (i.e. ici des frames)\n",
        "i2label = list(set(all_labels_strs))\n",
        "label2i = {x:i for i,x in enumerate(i2label)}\n",
        "# l'id du frame spécial \"Other_sense\"\n",
        "i_OTHER_SENSE = label2i['Other_sense']\n",
        "\n",
        "# séquence des ids de labels gold \n",
        "# pour chaque sous-corpus (clé = partie de corpus dev/train/test/val)\n",
        "labels = {}\n",
        "for p in label_strs.keys():\n",
        "    labels[p] = [label2i[x] for x in label_strs[p]]\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Longueur max des phrases: {'dev': 115, 'train': 271, 'test': 140}\n",
            "Rang max du target (en mots): {'dev': 96, 'train': 267, 'test': 115}\n",
            "SPLIT 18657 items into 16792 and 1865\n",
            "SPLIT 18657 items into 16792 and 1865\n",
            "SPLIT 18657 items into 16792 and 1865\n",
            "SPLIT 18657 items into 16792 and 1865\n",
            "dev : 2688 sentences, average lentgh=38.03\n",
            "train : 16792 sentences, average lentgh=38.99\n",
            "test : 3447 sentences, average lentgh=38.45\n",
            "val : 1865 sentences, average lentgh=38.97\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "edb6wB9dfN2P",
        "outputId": "6d559996-683c-4a85-9164-79c2f7a7d3be"
      },
      "source": [
        "i_OTHER_SENSE"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "76"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "label2i"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OTPsIElZAWO3",
        "outputId": "45c8001f-db87-49ad-c91e-f12649cf3c43"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'FR_Expressing_side': 0,\n",
              " 'Willingness': 1,\n",
              " 'FR_Purpose': 2,\n",
              " 'FR_Hail': 3,\n",
              " 'Spelling_and_pronouncing': 4,\n",
              " 'FR_Reason': 5,\n",
              " 'FR_Judgment_communication': 6,\n",
              " 'Religious_belief': 7,\n",
              " 'FR_Cognizer_affecting.to_do': 8,\n",
              " 'FR_Being_named': 9,\n",
              " 'FR_Spending': 10,\n",
              " 'FR_Agree_or_refuse_to_act_in_favor_of': 11,\n",
              " 'FR_Means_for_purpose': 12,\n",
              " 'FR_Having_commercial_agreement': 13,\n",
              " 'FR_Influence_of_event_on_cognizer-Subjective_influence': 14,\n",
              " 'Quarreling': 15,\n",
              " 'FR_Attributed_information': 16,\n",
              " 'Causation': 17,\n",
              " 'FR_Attempt_supported_suasion': 18,\n",
              " 'FR_Getting_money': 19,\n",
              " 'FR_Encoding': 20,\n",
              " 'Response': 21,\n",
              " 'FR_Commitment.veracity': 22,\n",
              " 'FR_Memory-Remembering_experience-Remembering_information': 23,\n",
              " 'FR_Commercial_transaction': 24,\n",
              " 'Complaining': 25,\n",
              " 'FR_Cause_to_start-Launch_process': 26,\n",
              " 'FR_Quoting': 27,\n",
              " 'FR_Attempt_suasion.to_do': 28,\n",
              " 'FR_Being_in_favor_of': 29,\n",
              " 'Regard': 30,\n",
              " 'FR_Renunciation': 31,\n",
              " 'FR_Cause_earning': 32,\n",
              " 'Renting': 33,\n",
              " 'Referring_by_name': 34,\n",
              " 'FR_Cognizer_affecting.veracity': 35,\n",
              " 'Renting_out': 36,\n",
              " 'Be_in_agreement_on_assessment': 37,\n",
              " 'FR_Grant_permission-Permitting': 38,\n",
              " 'FR_Making_speech': 39,\n",
              " 'Other_sense': 40,\n",
              " 'FR_Speak_on_topic': 41,\n",
              " 'Explaining_the_facts': 42,\n",
              " 'Evidence': 43,\n",
              " 'Carry_goods': 44,\n",
              " 'Commerce_pay': 45,\n",
              " 'FR_Telling': 46,\n",
              " 'FR_Contingency-Objective_influence': 47,\n",
              " 'FR_Make_agreement_on_assessment': 48,\n",
              " 'FR_Expectation': 49,\n",
              " 'FR_Agree_or_refuse_to_act': 50,\n",
              " 'Convey_importance': 51,\n",
              " 'Make_agreement_on_action': 52,\n",
              " 'Name_conferral': 53,\n",
              " 'FR_Reimbursement': 54,\n",
              " 'Judgment': 55,\n",
              " 'Reveal_secret': 56,\n",
              " 'Questioning': 57,\n",
              " 'FR_Attempt_suasion.legitimacy': 58,\n",
              " 'FR_Statement-manner-noise': 59,\n",
              " 'FR_Reliance_on_expectation': 60,\n",
              " 'FR_Support_verb': 61,\n",
              " 'FR_Contacting': 62,\n",
              " 'FR_Attempt_suasion.veracity': 63,\n",
              " 'FR_Expressing_decision': 64,\n",
              " 'FR_Taking_sides': 65,\n",
              " 'FR_Cognizer_affecting.legitimacy': 66,\n",
              " 'FR_Commitment.to_do': 67,\n",
              " 'Make_possible_to_do': 68,\n",
              " 'Evoking': 69,\n",
              " 'FR_Judgment_communication_no_reason': 70,\n",
              " 'Adducing': 71,\n",
              " 'FR_Deciding': 72,\n",
              " 'Ratification': 73,\n",
              " 'Be_in_agreement_on_action': 74,\n",
              " 'FR_Proving': 75,\n",
              " 'Communication_response': 76,\n",
              " 'Judgment_direct_address': 77,\n",
              " 'Preventing': 78,\n",
              " 'Summarizing': 79,\n",
              " 'Predicting': 80,\n",
              " 'Coming_to_believe': 81,\n",
              " 'Importing': 82,\n",
              " 'FR_Deserving': 83,\n",
              " 'FR_Request': 84,\n",
              " 'Appointing': 85,\n",
              " 'FR_Cause_enunciation': 86,\n",
              " 'Commerce_buy': 87,\n",
              " 'Remembering_to_do': 88,\n",
              " 'Bragging': 89,\n",
              " 'FR_Expressing_truth_of_proposition': 90,\n",
              " 'FR_Justifying': 91,\n",
              " 'Commerce_collect': 92,\n",
              " 'FR_Giving_money': 93,\n",
              " 'Exporting': 94,\n",
              " 'Categorization': 95,\n",
              " 'FR_Becoming_aware': 96,\n",
              " 'FR_Attributing_cause': 97,\n",
              " 'FR_Awareness-Certainty-Opinion': 98,\n",
              " 'FR_Commerce_scenario': 99,\n",
              " 'Respond_to_proposal': 100,\n",
              " 'Repayment': 101,\n",
              " 'Text_creation': 102,\n",
              " 'Commerce_sell': 103,\n",
              " 'Earnings_and_losses': 104,\n",
              " 'FR_Chatting-Discussion': 105}"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CIM0NNOQ9PHY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e8a997b-172b-4ddd-daf6-0ba157d7ca30"
      },
      "source": [
        "lemma2i"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'prévisible': 0,\n",
              " 'revente': 1,\n",
              " 'envisager': 2,\n",
              " 'assurer': 3,\n",
              " 'persuasion': 4,\n",
              " 'avis': 5,\n",
              " 'sentir': 6,\n",
              " 'tabler': 7,\n",
              " 'songer': 8,\n",
              " 'croire': 9,\n",
              " 'compréhension': 10,\n",
              " 'se_résumer': 11,\n",
              " 'dépense': 12,\n",
              " 'réaction': 13,\n",
              " 'devoir': 14,\n",
              " 'à_force_de': 15,\n",
              " 'implorer': 16,\n",
              " 'baptiser': 17,\n",
              " 'indiquer': 18,\n",
              " 'demander': 19,\n",
              " 'moteur': 20,\n",
              " 'reprise': 21,\n",
              " 'régler': 22,\n",
              " 'il_inspirer': 23,\n",
              " 'décrier': 24,\n",
              " 'commencer': 25,\n",
              " 'brouille': 26,\n",
              " 'constatation': 27,\n",
              " 'afin_que': 28,\n",
              " 'unanime': 29,\n",
              " 'dialoguer': 30,\n",
              " 'conscience': 31,\n",
              " 'conduire': 32,\n",
              " 'facteur': 33,\n",
              " 'oublier': 34,\n",
              " 'souligner': 35,\n",
              " 'enjoindre': 36,\n",
              " 'balbutier': 37,\n",
              " \"représentant_de_l'ordre\": 38,\n",
              " 'donner_naissance': 39,\n",
              " 'désavouer': 40,\n",
              " 'noter': 41,\n",
              " 'persuader': 42,\n",
              " 'décidé': 43,\n",
              " 'vanter': 44,\n",
              " 'se_plaindre': 45,\n",
              " 'écouler': 46,\n",
              " 'vu': 47,\n",
              " 'écrit': 48,\n",
              " 'validation': 49,\n",
              " 'informer': 50,\n",
              " 'impression': 51,\n",
              " 'faire_confiance': 52,\n",
              " 'étant_donné': 53,\n",
              " 'résumé': 54,\n",
              " 'suite': 55,\n",
              " 'confesser': 56,\n",
              " 'déceler': 57,\n",
              " 'convaincu': 58,\n",
              " 'faire_bloc': 59,\n",
              " 'mener_à_bien': 60,\n",
              " 'par_la_faute_de': 61,\n",
              " 'apporter': 62,\n",
              " 'trouver': 63,\n",
              " 'classification': 64,\n",
              " 'joindre': 65,\n",
              " 'nationalisation': 66,\n",
              " 'désaveu': 67,\n",
              " 'vente': 68,\n",
              " 'imputable': 69,\n",
              " 'exigence': 70,\n",
              " 'riposter': 71,\n",
              " 'contact': 72,\n",
              " 'plaidoirie': 73,\n",
              " \"d'après\": 74,\n",
              " 'faire_appel': 75,\n",
              " 'résolution': 76,\n",
              " 'dans_quel_but': 77,\n",
              " 'jugement': 78,\n",
              " 'si_bien_que': 79,\n",
              " 'symptôme': 80,\n",
              " 'rétribution': 81,\n",
              " 'divisés': 82,\n",
              " 'presser': 83,\n",
              " 'observer': 84,\n",
              " 'indice': 85,\n",
              " 'conseiller': 86,\n",
              " 'conséquent': 87,\n",
              " 'faire_remarquer': 88,\n",
              " 'but': 89,\n",
              " 'par_suite': 90,\n",
              " 'soulever': 91,\n",
              " 'hypothèse': 92,\n",
              " 'disputer': 93,\n",
              " 'à_raison_de': 94,\n",
              " 'en_effet': 95,\n",
              " 'voir': 96,\n",
              " 'se_disputer': 97,\n",
              " 'interprétation': 98,\n",
              " 'mépris': 99,\n",
              " 'démentir': 100,\n",
              " \"voir_d'un_mauvais_oeil\": 101,\n",
              " 'objection': 102,\n",
              " 'cession': 103,\n",
              " \"s'indigner\": 104,\n",
              " 'contrecoup': 105,\n",
              " 'escompter': 106,\n",
              " 'louange': 107,\n",
              " 'chanter': 108,\n",
              " 'divergence': 109,\n",
              " 'dialogue_de_sourd': 110,\n",
              " 'aussi': 111,\n",
              " 'sermon': 112,\n",
              " 'rendre': 113,\n",
              " 'ordre': 114,\n",
              " 'fonction': 115,\n",
              " 'dits': 116,\n",
              " 'remercier': 117,\n",
              " 'au_fait': 118,\n",
              " 'préciser': 119,\n",
              " 'pour_ainsi_dire': 120,\n",
              " 'interpeller': 121,\n",
              " 'deviner': 122,\n",
              " 'alors': 123,\n",
              " 'aborder': 124,\n",
              " 'produire': 125,\n",
              " 'décourager': 126,\n",
              " 'désaccord': 127,\n",
              " 'sensibiliser': 128,\n",
              " 'acquisition': 129,\n",
              " 'déplorer': 130,\n",
              " 'tant_et_si_bien_que': 131,\n",
              " 'faire_savoir': 132,\n",
              " 'engager': 133,\n",
              " 'pourparler': 134,\n",
              " 'cotiser': 135,\n",
              " 'disposé': 136,\n",
              " 'répliquer': 137,\n",
              " 'opposition': 138,\n",
              " 'pour_que': 139,\n",
              " 'projet': 140,\n",
              " 'contrôle': 141,\n",
              " 'douteux': 142,\n",
              " 'confier': 143,\n",
              " 'anathème': 144,\n",
              " 'ralliement': 145,\n",
              " 'percevoir': 146,\n",
              " 'assuré': 147,\n",
              " 'reproche': 148,\n",
              " 'nationaliser': 149,\n",
              " 'nomination': 150,\n",
              " 'alerter': 151,\n",
              " 'certain': 152,\n",
              " 'alléguer': 153,\n",
              " 'connaissance': 154,\n",
              " 'déduction': 155,\n",
              " 'dites': 156,\n",
              " 'prendre_le_parti': 157,\n",
              " 'contacter': 158,\n",
              " 'souscrire': 159,\n",
              " 'supposer': 160,\n",
              " 'convertir': 161,\n",
              " 'demande': 162,\n",
              " 'porter_-_parole': 163,\n",
              " 'haranguer': 164,\n",
              " \"s'aviser\": 165,\n",
              " 'avancer': 166,\n",
              " 'reversement': 167,\n",
              " 'faire_en_sorte': 168,\n",
              " 'causerie': 169,\n",
              " 'racheter': 170,\n",
              " 'signe': 171,\n",
              " 'montrer': 172,\n",
              " \"se_mettre_d'accord\": 173,\n",
              " 'engueulade': 174,\n",
              " 'convenir': 175,\n",
              " 'classement': 176,\n",
              " 'perte': 177,\n",
              " 'refuser': 178,\n",
              " 'sentiment': 179,\n",
              " 'à_cause_de': 180,\n",
              " 'justifier': 181,\n",
              " 'remettre': 182,\n",
              " 'parler': 183,\n",
              " 'regretter': 184,\n",
              " 'reverser': 185,\n",
              " 'inspirer': 186,\n",
              " 'faire_son_apparition': 187,\n",
              " 'se_rendre': 188,\n",
              " 'souvenir': 189,\n",
              " 'défendre': 190,\n",
              " 'infirmer': 191,\n",
              " 'engendrer': 192,\n",
              " 'complimenter': 193,\n",
              " 'négoce': 194,\n",
              " 'pronostic': 195,\n",
              " 'répéter': 196,\n",
              " 'sur_la_demande_de': 197,\n",
              " 'commercer': 198,\n",
              " 'ordonner': 199,\n",
              " 'aboutissement': 200,\n",
              " 'se_défendre': 201,\n",
              " 'convier': 202,\n",
              " 'nommé': 203,\n",
              " 'prononcer': 204,\n",
              " 'critique': 205,\n",
              " 'soutenir': 206,\n",
              " 'mépriser': 207,\n",
              " 'à_cet_effet': 208,\n",
              " 'convergence': 209,\n",
              " 'contrainte': 210,\n",
              " 'recouvrement': 211,\n",
              " 'opposer': 212,\n",
              " 'indication': 213,\n",
              " 'impulser': 214,\n",
              " 'précision': 215,\n",
              " 'amener': 216,\n",
              " 'promulguer': 217,\n",
              " 'admirer': 218,\n",
              " 'permission': 219,\n",
              " 'prélever': 220,\n",
              " 'conter': 221,\n",
              " 'influence': 222,\n",
              " 'exalter': 223,\n",
              " 'dialogue': 224,\n",
              " 'renchérir': 225,\n",
              " 'jurer': 226,\n",
              " 'exiger': 227,\n",
              " 'déchaîner': 228,\n",
              " 'transaction': 229,\n",
              " 'menace': 230,\n",
              " 'comme': 231,\n",
              " 'vision': 232,\n",
              " 'ambitionner': 233,\n",
              " 'gager': 234,\n",
              " 'reconnaître': 235,\n",
              " 'promettre': 236,\n",
              " 'attester': 237,\n",
              " 'écrire': 238,\n",
              " 'faire_référence': 239,\n",
              " 'féliciter': 240,\n",
              " 'dépendre': 241,\n",
              " 'commenter': 242,\n",
              " 'faire_ses_preuves': 243,\n",
              " \"force_de_l'ordre\": 244,\n",
              " \"s'exclamer\": 245,\n",
              " 'déni': 246,\n",
              " 'prédire': 247,\n",
              " 'étant_donné_que': 248,\n",
              " 'prendre_sa_source': 249,\n",
              " 'initiateur': 250,\n",
              " 'cotisation': 251,\n",
              " 'laisser': 252,\n",
              " 'inciter': 253,\n",
              " 'accepter': 254,\n",
              " 'reconnaissance': 255,\n",
              " 'stimuler': 256,\n",
              " 'causer': 257,\n",
              " 'disposer': 258,\n",
              " 'à_la_suite_de': 259,\n",
              " 'admettre': 260,\n",
              " 'argument': 261,\n",
              " 'en_accord_avec': 262,\n",
              " 'jouer': 263,\n",
              " 'acceptation': 264,\n",
              " 'intuition': 265,\n",
              " 'discuter': 266,\n",
              " 'nommer': 267,\n",
              " 'constater': 268,\n",
              " 'témoignage': 269,\n",
              " 'ferrailler': 270,\n",
              " 'impact': 271,\n",
              " 'en_retour': 272,\n",
              " 'vouloir_dire': 273,\n",
              " 'contredire': 274,\n",
              " 'réponse': 275,\n",
              " 'infliger': 276,\n",
              " 'import': 277,\n",
              " 'vue': 278,\n",
              " 'contre': 279,\n",
              " 'favorable': 280,\n",
              " 'porter_parole': 281,\n",
              " 'impliquer': 282,\n",
              " 'étiquetage': 283,\n",
              " 'crier': 284,\n",
              " 'faire_défaut': 285,\n",
              " 'débat': 286,\n",
              " 'exposer': 287,\n",
              " 'adhésion': 288,\n",
              " 'en_désaccord': 289,\n",
              " 'à_sa_demande': 290,\n",
              " 'mention': 291,\n",
              " 'rapporter': 292,\n",
              " 'en_vertu_de': 293,\n",
              " 'évoquer': 294,\n",
              " 'du_fait_de': 295,\n",
              " 'contradiction': 296,\n",
              " 'malédiction': 297,\n",
              " 'rémunération': 298,\n",
              " 'et_pour_cause': 299,\n",
              " 'brader': 300,\n",
              " \"avec_l'objectif_de\": 301,\n",
              " 'révélation': 302,\n",
              " 'incertitude': 303,\n",
              " 'pour_cause_de': 304,\n",
              " 'engagement': 305,\n",
              " 'questionner': 306,\n",
              " 'donneur_ordre': 307,\n",
              " 'exhorter': 308,\n",
              " 'accentuer': 309,\n",
              " 'de_fait': 310,\n",
              " 'du_fait_que': 311,\n",
              " 'alerte': 312,\n",
              " 'certifier': 313,\n",
              " 'donner_son_accord': 314,\n",
              " 'délibérer': 315,\n",
              " 'se_rendre_compte': 316,\n",
              " 'délibération': 317,\n",
              " 'discrédit': 318,\n",
              " 'produit': 319,\n",
              " 'entamer': 320,\n",
              " 'alimenter': 321,\n",
              " 'induire': 322,\n",
              " 'contester': 323,\n",
              " 'pressentir': 324,\n",
              " 'acquitter': 325,\n",
              " 'sensibilisation': 326,\n",
              " 'payer': 327,\n",
              " 'importer': 328,\n",
              " 'incertain': 329,\n",
              " 'dénoncer': 330,\n",
              " 'à_juste_raison': 331,\n",
              " 'voir_le_jour': 332,\n",
              " 'obliger': 333,\n",
              " 'susciter': 334,\n",
              " 'objecter': 335,\n",
              " 'soutien': 336,\n",
              " 'influencer': 337,\n",
              " 'chant': 338,\n",
              " 'péage': 339,\n",
              " 'dépenser': 340,\n",
              " 'en_conséquence': 341,\n",
              " 'invitation': 342,\n",
              " 'commerce': 343,\n",
              " 'provoquer': 344,\n",
              " 'se_lamenter': 345,\n",
              " 'penser': 346,\n",
              " 'avertir': 347,\n",
              " 'en_ordre': 348,\n",
              " 'contre-attaque': 349,\n",
              " 'serment': 350,\n",
              " 'avaliser': 351,\n",
              " 'en_bon_ordre': 352,\n",
              " 'alliance': 353,\n",
              " 'commercialiser': 354,\n",
              " 'riposte': 355,\n",
              " 'réclamation': 356,\n",
              " 'marque': 357,\n",
              " 'assurance': 358,\n",
              " 'autorisation': 359,\n",
              " 'propos': 360,\n",
              " 'accréditer': 361,\n",
              " 'mettre_sur_le_compte_de': 362,\n",
              " 'divulgation': 363,\n",
              " 'animer': 364,\n",
              " 'inattendu': 365,\n",
              " 'acculer': 366,\n",
              " 'anticiper': 367,\n",
              " 'permettre': 368,\n",
              " 'distribution': 369,\n",
              " 'contestation': 370,\n",
              " 'relater': 371,\n",
              " 'paraître': 372,\n",
              " 'flatter': 373,\n",
              " 'dédain': 374,\n",
              " 'dit': 375,\n",
              " 'dire': 376,\n",
              " 'classifier': 377,\n",
              " 'citation': 378,\n",
              " 'protester': 379,\n",
              " 'appeler': 380,\n",
              " 'déposition': 381,\n",
              " \"raison_d'être\": 382,\n",
              " 'fruit': 383,\n",
              " 'prévision': 384,\n",
              " 'mentionner': 385,\n",
              " 'avertissement': 386,\n",
              " 'dû': 387,\n",
              " 'stimulation': 388,\n",
              " 'se_faire_prier': 389,\n",
              " 'rejeter': 390,\n",
              " \"s'accorder\": 391,\n",
              " 'manifestation': 392,\n",
              " 'du_coup': 393,\n",
              " 'requérir': 394,\n",
              " 'résoudre': 395,\n",
              " 'générateur': 396,\n",
              " 'induction': 397,\n",
              " 'preuve': 398,\n",
              " 'mener': 399,\n",
              " 'en_appeler': 400,\n",
              " 'donc': 401,\n",
              " \"donneur_d'ordre\": 402,\n",
              " 'achat': 403,\n",
              " 'commentaire': 404,\n",
              " 'amodiation': 405,\n",
              " 'prêt': 406,\n",
              " 'considérer': 407,\n",
              " \"jusqu'à\": 408,\n",
              " 'défense': 409,\n",
              " 'promesse': 410,\n",
              " 'argumentation': 411,\n",
              " 'remarquer': 412,\n",
              " 'remboursement': 413,\n",
              " 'credo': 414,\n",
              " \"raison_d'état\": 415,\n",
              " 'renoncer': 416,\n",
              " 'applaudir': 417,\n",
              " 'doute': 418,\n",
              " 'merci': 419,\n",
              " 'connaître': 420,\n",
              " 'décliner': 421,\n",
              " 'privatiser': 422,\n",
              " 'pensée': 423,\n",
              " \"voir_d'un_oeil_bienveillant\": 424,\n",
              " 'maudire': 425,\n",
              " 'éviter': 426,\n",
              " 'affirmation': 427,\n",
              " 'écriture': 428,\n",
              " 'palabre': 429,\n",
              " 'point_de_vue': 430,\n",
              " 'condamner': 431,\n",
              " 'repousser': 432,\n",
              " \"à_l'appel_de\": 433,\n",
              " 'ronchonner': 434,\n",
              " 'aviser': 435,\n",
              " 'recouvrer': 436,\n",
              " 'se_confier': 437,\n",
              " 'concevoir': 438,\n",
              " 'export': 439,\n",
              " 'confirmer': 440,\n",
              " \"d'avis\": 441,\n",
              " 'avoir_raison': 442,\n",
              " 'fins': 443,\n",
              " 'donner_raison': 444,\n",
              " 'grâce_à': 445,\n",
              " 'communication': 446,\n",
              " 'confession': 447,\n",
              " 'ordre_public': 448,\n",
              " 'à_point_nommé': 449,\n",
              " 'solliciter': 450,\n",
              " 'paie': 451,\n",
              " 'apprendre': 452,\n",
              " \"s'appeler\": 453,\n",
              " 'au_mépris_de': 454,\n",
              " 'proclamer': 455,\n",
              " 'dispute': 456,\n",
              " 'proférer': 457,\n",
              " 'prévenir': 458,\n",
              " 'porter_le_à': 459,\n",
              " 'injonction': 460,\n",
              " 'se_rallier': 461,\n",
              " 'notifier': 462,\n",
              " 'compte-rendu': 463,\n",
              " 'révéler': 464,\n",
              " 'faire_ses_courses': 465,\n",
              " 'découverte': 466,\n",
              " 'au_point_que': 467,\n",
              " 'prier': 468,\n",
              " 'atténuer': 469,\n",
              " 'louer': 470,\n",
              " 'en_coûter': 471,\n",
              " 'faire_face': 472,\n",
              " 'déclaration': 473,\n",
              " 'surcoût': 474,\n",
              " 'renvoyer_la_balle': 475,\n",
              " 'effet': 476,\n",
              " 'requête': 477,\n",
              " 'plus_que_de_raison': 478,\n",
              " 'affecter': 479,\n",
              " 'allusion': 480,\n",
              " 'renonciation': 481,\n",
              " 'dès_lors': 482,\n",
              " 'réflexe': 483,\n",
              " 'discréditer': 484,\n",
              " 'sur_ordre_de': 485,\n",
              " 'toucher': 486,\n",
              " 'se_justifier': 487,\n",
              " 'se_perdre': 488,\n",
              " 'en_vertu_duquel': 489,\n",
              " 'en_vue_de': 490,\n",
              " 'verser': 491,\n",
              " 'donner_à_croire': 492,\n",
              " 'encouragement': 493,\n",
              " 'réaliser': 494,\n",
              " 'prendre_la_parole': 495,\n",
              " 'nier': 496,\n",
              " 'déduire': 497,\n",
              " 'cataloguer': 498,\n",
              " 'reparler': 499,\n",
              " 'ajouter': 500,\n",
              " 'faire_sentir': 501,\n",
              " \"rappeler_à_l'ordre\": 502,\n",
              " 'sollicitation': 503,\n",
              " 'menacer': 504,\n",
              " 'liquider': 505,\n",
              " 'encourager': 506,\n",
              " 'étayer': 507,\n",
              " 'sommation': 508,\n",
              " \"s'allier\": 509,\n",
              " 'faire_part': 510,\n",
              " 'se_flatter': 511,\n",
              " \"s'inspirer\": 512,\n",
              " 'chanter_les_louanges': 513,\n",
              " 'exaltation': 514,\n",
              " 'facturation': 515,\n",
              " 'constat': 516,\n",
              " 'préjuger': 517,\n",
              " 'faire_valoir': 518,\n",
              " 'détecter': 519,\n",
              " 'interroger': 520,\n",
              " \"voir_d'un_bon_oeil\": 521,\n",
              " 'résolu': 522,\n",
              " 'prédiction': 523,\n",
              " 'compter': 524,\n",
              " 'infléchir': 525,\n",
              " 'affréter': 526,\n",
              " 'ambition': 527,\n",
              " 'parier': 528,\n",
              " 'rejet': 529,\n",
              " 'pour': 530,\n",
              " 'certitude': 531,\n",
              " 'soupçon': 532,\n",
              " 'déterminant': 533,\n",
              " 'ainsi': 534,\n",
              " 'conclure': 535,\n",
              " 'explication': 536,\n",
              " 'empêcher': 537,\n",
              " \"mot_d'ordre\": 538,\n",
              " 'rapport': 539,\n",
              " 'par_conséquent': 540,\n",
              " 'dessein': 541,\n",
              " \"c'est_pourquoi\": 542,\n",
              " 'atteindre': 543,\n",
              " \"s'entendre\": 544,\n",
              " 'suggestion': 545,\n",
              " 'dès_lors_que': 546,\n",
              " 'dissuader': 547,\n",
              " 'issue': 548,\n",
              " 'encaisser': 549,\n",
              " 'téléphoner': 550,\n",
              " 'foi': 551,\n",
              " 'témoigner': 552,\n",
              " 'applaudissement': 553,\n",
              " 'prendre_position': 554,\n",
              " 'proclamation': 555,\n",
              " 'entraînement': 556,\n",
              " 'perdre': 557,\n",
              " 'estimer': 558,\n",
              " \"s'adresser\": 559,\n",
              " 'de_façon_à': 560,\n",
              " 'à_preuve': 561,\n",
              " 'sceptique': 562,\n",
              " 'se_décider': 563,\n",
              " 'argumenter': 564,\n",
              " 'corroborer': 565,\n",
              " 'accuser': 566,\n",
              " 'contrer': 567,\n",
              " 'raconter': 568,\n",
              " 'rembourser': 569,\n",
              " 'divulguer': 570,\n",
              " 'réagir': 571,\n",
              " 'lancement': 572,\n",
              " 'symptomatique': 573,\n",
              " 'subir': 574,\n",
              " 'impulsion': 575,\n",
              " 'tolérer': 576,\n",
              " 'refléter': 577,\n",
              " 'pour_preuve': 578,\n",
              " 'illusion': 579,\n",
              " 'déprécier': 580,\n",
              " 'grogner': 581,\n",
              " 'à_tel_point_que': 582,\n",
              " 'reprocher': 583,\n",
              " \"de_l'ordre_de\": 584,\n",
              " 'effectivement': 585,\n",
              " 'conviction': 586,\n",
              " 'découler': 587,\n",
              " 'dissuasion': 588,\n",
              " 'autoriser': 589,\n",
              " 'perception': 590,\n",
              " 'faire_observer': 591,\n",
              " 'faire_un_geste': 592,\n",
              " 'faire': 593,\n",
              " 'tenir': 594,\n",
              " 'compliment': 595,\n",
              " 'incidence': 596,\n",
              " 'savoir': 597,\n",
              " 'solde': 598,\n",
              " 'à_défaut_de': 599,\n",
              " 'répondre': 600,\n",
              " 'réplique': 601,\n",
              " 'se_faire_attendre': 602,\n",
              " 'éprouver': 603,\n",
              " 'gémir': 604,\n",
              " 'imputer': 605,\n",
              " 'expression': 606,\n",
              " 'allumer': 607,\n",
              " 'pronostiquer': 608,\n",
              " 'discourir': 609,\n",
              " 'inviter': 610,\n",
              " \"c'est_que\": 611,\n",
              " 'dénier': 612,\n",
              " 'préjugé': 613,\n",
              " 'dans_un_but_de': 614,\n",
              " 'créer': 615,\n",
              " 'aveu': 616,\n",
              " 'affirmer': 617,\n",
              " 'dédaigner': 618,\n",
              " 'motif': 619,\n",
              " 'à_ce_point_que': 620,\n",
              " 'congratuler': 621,\n",
              " 'à_proprement_parler': 622,\n",
              " \"sous_l'effet_de\": 623,\n",
              " 'estime': 624,\n",
              " 'retombée': 625,\n",
              " 'formuler': 626,\n",
              " 'conseil': 627,\n",
              " 'inconnu': 628,\n",
              " \"c'est_-_à_-_dire\": 629,\n",
              " 'décision': 630,\n",
              " 'frais': 631,\n",
              " 'prière': 632,\n",
              " 'aboutir': 633,\n",
              " 'paye': 634,\n",
              " 'trancher': 635,\n",
              " 'rajouter': 636,\n",
              " 'influençable': 637,\n",
              " 'contrôler': 638,\n",
              " 'approbation': 639,\n",
              " 'récuser': 640,\n",
              " 'dans_la_mesure_où': 641,\n",
              " 'cause': 642,\n",
              " 'laisser_entendre': 643,\n",
              " 'écoulement': 644,\n",
              " 'se_féliciter': 645,\n",
              " 'de_ce_fait': 646,\n",
              " 'dans_ce_but': 647,\n",
              " 'débourser': 648,\n",
              " 'rachat': 649,\n",
              " 'accusation': 650,\n",
              " 'subodorer': 651,\n",
              " 'donner_à_penser': 652,\n",
              " 'entrevoir': 653,\n",
              " 'pourquoi': 654,\n",
              " 'conception': 655,\n",
              " 'afin_de': 656,\n",
              " 'origine': 657,\n",
              " 'preuve_que': 658,\n",
              " 'dénigrement': 659,\n",
              " 'faire_suite': 660,\n",
              " 'amorcer': 661,\n",
              " 'prélèvement': 662,\n",
              " 'marquer': 663,\n",
              " 'se_convertir': 664,\n",
              " 'protestation': 665,\n",
              " 'entraîner': 666,\n",
              " 'location': 667,\n",
              " 'renoncement': 668,\n",
              " 'ouvrir': 669,\n",
              " \"d'où\": 670,\n",
              " 'bonification': 671,\n",
              " 'détailler': 672,\n",
              " 'sinon': 673,\n",
              " \"s'élever\": 674,\n",
              " 'conversion': 675,\n",
              " 'réfuter': 676,\n",
              " 'prise_de_conscience': 677,\n",
              " 'diviser': 678,\n",
              " 'exporter': 679,\n",
              " 'légitimer': 680,\n",
              " 'débuter': 681,\n",
              " 'illustrer': 682,\n",
              " 'mobile': 683,\n",
              " 'démarrer': 684,\n",
              " \"s'apercevoir\": 685,\n",
              " 'se_rappeler': 686,\n",
              " 'solder': 687,\n",
              " 'acquérir': 688,\n",
              " 'occasionner': 689,\n",
              " 'suggérer': 690,\n",
              " 'à_son_avis': 691,\n",
              " 'présomption': 692,\n",
              " 'faire_état': 693,\n",
              " 'imprévu': 694,\n",
              " 'témoin': 695,\n",
              " 'rendre_son_tablier': 696,\n",
              " 'douter': 697,\n",
              " 'plaider': 698,\n",
              " 'à_vrai_dire': 699,\n",
              " 'recommander': 700,\n",
              " \"s'engager\": 701,\n",
              " \"s'expliquer\": 702,\n",
              " 'de_manière_à': 703,\n",
              " 'coup_de_fil': 704,\n",
              " 'opinion': 705,\n",
              " 'commercialisation': 706,\n",
              " 'dénonciation': 707,\n",
              " 'ce_être_-_à_-_dire': 708,\n",
              " 'remarque': 709,\n",
              " 'découvrir': 710,\n",
              " 'sachant_que': 711,\n",
              " \"c'est_à_-_dire\": 712,\n",
              " 'présager': 713,\n",
              " 'rallier': 714,\n",
              " 'gain': 715,\n",
              " 'lentille_de_contact': 716,\n",
              " 'ignorance': 717,\n",
              " 'création': 718,\n",
              " 'formulation': 719,\n",
              " 'par_contrecoup': 720,\n",
              " 'au_courant': 721,\n",
              " 'ressort': 722,\n",
              " 'censé': 723,\n",
              " 'repérer': 724,\n",
              " 'promulgation': 725,\n",
              " 'contraindre': 726,\n",
              " 'relever': 727,\n",
              " 'réveiller': 728,\n",
              " 'admiration': 729,\n",
              " 'imprévisible': 730,\n",
              " 'du': 731,\n",
              " 'lancer': 732,\n",
              " 'interdire': 733,\n",
              " 'conclusion': 734,\n",
              " 'faire_preuve': 735,\n",
              " 'inspiration': 736,\n",
              " 'marmonner': 737,\n",
              " 'déclenchement': 738,\n",
              " 'au_motif_que': 739,\n",
              " \"s'arranger\": 740,\n",
              " 'présumer': 741,\n",
              " 'plaidoyer': 742,\n",
              " 'fustiger': 743,\n",
              " 'évocation': 744,\n",
              " 'versement': 745,\n",
              " 'raison': 746,\n",
              " 'dans_cet_objectif': 747,\n",
              " 'justification': 748,\n",
              " 'vouloir_pour_preuve': 749,\n",
              " 'consentement': 750,\n",
              " 'dicter': 751,\n",
              " 'interpréter': 752,\n",
              " 'apprécier': 753,\n",
              " 'se_concerter': 754,\n",
              " 'perdre_connaissance': 755,\n",
              " 'désapprouver': 756,\n",
              " 'attirer': 757,\n",
              " 'préconiser': 758,\n",
              " 'condition': 759,\n",
              " 'conditionner': 760,\n",
              " 'professer': 761,\n",
              " 'concéder': 762,\n",
              " 'rétorquer': 763,\n",
              " 'remerciement': 764,\n",
              " 'être_fonction_de': 765,\n",
              " 'déclencher': 766,\n",
              " 'annonce': 767,\n",
              " 'débattre': 768,\n",
              " 'influer': 769,\n",
              " 'exposé': 770,\n",
              " 'entendre': 771,\n",
              " 'prôner': 772,\n",
              " 'se_prononcer': 773,\n",
              " 'convaincre': 774,\n",
              " 'miser': 775,\n",
              " 'à_des_fins': 776,\n",
              " 'imaginer': 777,\n",
              " 'juger': 778,\n",
              " 'source': 779,\n",
              " 'naître': 780,\n",
              " 'importation': 781,\n",
              " 'partisan': 782,\n",
              " 'prendre_connaissance': 783,\n",
              " 'motivation': 784,\n",
              " 'gronder': 785,\n",
              " 'porter_à_à': 786,\n",
              " 'avouer': 787,\n",
              " 'invalider': 788,\n",
              " 'revendre': 789,\n",
              " 'consentir': 790,\n",
              " 'faute_de': 791,\n",
              " 'se_vanter': 792,\n",
              " 'tractation': 793,\n",
              " 'parce_que': 794,\n",
              " 'se_souvenir': 795,\n",
              " 'séquelle': 796,\n",
              " 'privatisation': 797,\n",
              " 'forcer': 798,\n",
              " 'interrogation': 799,\n",
              " 'de_sorte_que': 800,\n",
              " 'déclarer': 801,\n",
              " 'réticent': 802,\n",
              " 'à_des_fins_de': 803,\n",
              " 'se_douter': 804,\n",
              " \"sur_l'injonction_de\": 805,\n",
              " 'oubli': 806,\n",
              " 'initier': 807,\n",
              " 'sûr': 808,\n",
              " 'rendre_compte': 809,\n",
              " 'rappeler': 810,\n",
              " 'avoir_pour_nom': 811,\n",
              " 'responsable': 812,\n",
              " 'conscient': 813,\n",
              " \"s'insurger\": 814,\n",
              " \"dans_l'optique_de\": 815,\n",
              " \"s'exprimer\": 816,\n",
              " \"de_l'avis_de\": 817,\n",
              " 'prêcher': 818,\n",
              " 'annoncer': 819,\n",
              " 'puisque': 820,\n",
              " 'sommer': 821,\n",
              " 'réprimander': 822,\n",
              " 'porter': 823,\n",
              " 'marché': 824,\n",
              " 'résultat': 825,\n",
              " 'râler': 826,\n",
              " 'classer': 827,\n",
              " 'déterminer': 828,\n",
              " 'valider': 829,\n",
              " \"jusqu'à_ce_que\": 830,\n",
              " 'idée': 831,\n",
              " 'au_vu_de': 832,\n",
              " 'appel': 833,\n",
              " 'interdiction': 834,\n",
              " 'scepticisme': 835,\n",
              " 'se_targuer': 836,\n",
              " 'confirmation': 837,\n",
              " 'répétition': 838,\n",
              " \"protocole_d'accord\": 839,\n",
              " 'dans_le_but_de': 840,\n",
              " 'appelé': 841,\n",
              " 'coût': 842,\n",
              " 'motiver': 843,\n",
              " 'astreindre': 844,\n",
              " 'projeter': 845,\n",
              " 'désignation': 846,\n",
              " 'comprendre': 847,\n",
              " 'faire_attention': 848,\n",
              " \"s'opposer\": 849,\n",
              " 'exportation': 850,\n",
              " 'hommage': 851,\n",
              " 'bradage': 852,\n",
              " 'recommandation': 853,\n",
              " 'allocution': 854,\n",
              " 'épingler': 855,\n",
              " 'polémique': 856,\n",
              " 'au_point_de': 857,\n",
              " 'critiquer': 858,\n",
              " 'apostropher': 859,\n",
              " 'arrangement': 860,\n",
              " \"s'attendre\": 861,\n",
              " 'rémunérer': 862,\n",
              " 'croyance': 863,\n",
              " 'signaler': 864,\n",
              " 'générer': 865,\n",
              " 'être_ce_à_dire': 866,\n",
              " 'anticipation': 867,\n",
              " 'soupçonner': 868,\n",
              " 'exprimer': 869,\n",
              " 'intention': 870,\n",
              " 'suspecter': 871,\n",
              " \"à_l'origine_de\": 872,\n",
              " \"tomber_d'accord\": 873,\n",
              " 'convaincant': 874,\n",
              " 'désapprobation': 875,\n",
              " 'conversation': 876,\n",
              " 'légitimation': 877,\n",
              " 'conséquence': 878,\n",
              " 'sonder': 879,\n",
              " \"faire_l'objet\": 880,\n",
              " 'observation': 881,\n",
              " 'fonder': 882,\n",
              " 'refus': 883,\n",
              " 'appel_téléphonique': 884,\n",
              " 'prévention': 885,\n",
              " 'vu_que': 886,\n",
              " 'faire_naître': 887,\n",
              " 'céder': 888,\n",
              " 'attribuer': 889,\n",
              " 'communiquer': 890,\n",
              " 'distribuer': 891,\n",
              " 'résulter': 892,\n",
              " 'remémorer': 893,\n",
              " 'facturer': 894,\n",
              " 'ordre_du_jour': 895,\n",
              " 'par_voie_de_conséquence': 896,\n",
              " 'fondement': 897,\n",
              " 'atténuation': 898,\n",
              " 'expliquer': 899,\n",
              " 'condamnation': 900,\n",
              " 'ignorer': 901,\n",
              " 'reprendre': 902,\n",
              " 'élogieux': 903,\n",
              " 'récrimination': 904,\n",
              " 'se_résoudre': 905,\n",
              " 'entériner': 906,\n",
              " 'faire_partie': 907,\n",
              " 'résumer': 908,\n",
              " 'ménager': 909,\n",
              " 'à_la_demande_de': 910,\n",
              " 'gagner': 911,\n",
              " 'prouver': 912,\n",
              " 'coûter': 913,\n",
              " 'répercussion': 914,\n",
              " 'démontrer': 915,\n",
              " 'trouver_sa_source': 916,\n",
              " 'objectif': 917,\n",
              " 'divisées': 918,\n",
              " 'arguer': 919,\n",
              " 'pousser': 920,\n",
              " 'approuver': 921,\n",
              " 'citer': 922,\n",
              " 'acheter': 923,\n",
              " \"d'accord\": 924,\n",
              " 'hurler': 925,\n",
              " 'paiement': 926,\n",
              " 'bonifier': 927,\n",
              " 'manifester': 928,\n",
              " 'en_raison_de': 929,\n",
              " 'opposé': 930,\n",
              " 'attente': 931,\n",
              " 'dévoiler': 932,\n",
              " 'règlement': 933,\n",
              " 'décider': 934,\n",
              " 'indicateur': 935,\n",
              " 'avoir_à_revendre': 936,\n",
              " 'déterminé': 937,\n",
              " 'mettre_en_doute': 938,\n",
              " 'querelle': 939,\n",
              " 'selon': 940,\n",
              " 'incitation': 941,\n",
              " 'car': 942,\n",
              " 'adhérer': 943,\n",
              " 'prévoir': 944,\n",
              " 'réclamer': 945,\n",
              " 'discussion': 946,\n",
              " 'sembler': 947,\n",
              " 'vendre': 948,\n",
              " 'prétendre': 949,\n",
              " 'accord': 950,\n",
              " 'discours': 951,\n",
              " 'contrat': 952}"
            ]
          },
          "metadata": {},
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E2Rgiftb-CQ5"
      },
      "source": [
        "### Baseline MFS (\"most frequent sense\")"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2f7R9YD9_OH"
      },
      "source": [
        "# TODO:\n",
        "# calculer le sens le plus fréquent de chaque lemme-target\n",
        "# (le plus fréquent dans train+val)\n",
        "\n",
        "# et calculez la baseline MFS (\"most frequent sense\")\n",
        "# - sur le train+val\n",
        "# - sur le dev\n",
        "\n",
        "# TODO:\n",
        "# Etudiez les éléments de dev qui sont \"inconnus\" de train+val:\n",
        "# - les lemmes-target inconnus\n",
        "# - les frames inconnus\n",
        "# - les associations frame / lemme-target inconnus\n",
        "from collections import defaultdict\n",
        "def frequence(tg_lemmas,label_strs):\n",
        "  dict_lemmes = defaultdict(lambda: defaultdict(int))\n",
        "  dict_most_frequent = {}\n",
        "  for lemme, sens in zip(tg_lemmas,label_strs):\n",
        "      dict_lemmes[lemme][sens]+=1\n",
        "  for key, value in dict_lemmes.items():\n",
        "    max_item = max(value, key=lambda k: value[k])\n",
        "    dict_most_frequent[key] = max_item\n",
        "  return dict_lemmes, dict_most_frequent\n",
        "\n",
        "\n",
        "dict_lemmes_train, dict_most_frequent_train = frequence(tg_lemmas['train']+tg_lemmas['val'],label_strs['train']+label_strs['val'])\n",
        "dict_lemmes_val, dict_most_frequent_val = frequence(tg_lemmas['val'],label_strs['val'])\n",
        "dict_lemmes_dev, dict_most_frequent_dev = frequence(tg_lemmas['dev'],label_strs['dev'])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lUM5bg-4rDI3"
      },
      "source": [
        "def baseline(dict_most_frequent, tg_lemmas,label_strs,train_or_dev = 'train+val'):\n",
        "  amount_correct = 0\n",
        "  # This function will calculate the accuracy of baseline model\n",
        "  # based on naïve comparison with \"\"==\"\" of the most frequent sens\n",
        "  for lemme, sens in zip(tg_lemmas,label_strs):\n",
        "    if sens == dict_most_frequent[lemme]:\n",
        "      amount_correct +=1\n",
        "  return 'Accuracy of baseline model on ' + train_or_dev + ' is:',amount_correct/len(tg_lemmas)*100\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5qq0DQCJsmUd",
        "outputId": "1755e564-387d-4ac1-d787-0e6bde99c527"
      },
      "source": [
        "baseline(dict_most_frequent_train, tg_lemmas['train']+tg_lemmas['val'],label_strs['train']+label_strs['val'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Accuracy of baseline model on train+val is:', 81.39572278501367)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p79BXMSq7R5N",
        "outputId": "1480ad5b-d4b9-49d7-9477-a37194f1f10c"
      },
      "source": [
        "baseline(dict_most_frequent_dev, tg_lemmas['dev'],label_strs['dev'], 'dev')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Accuracy of baseline model on dev is:', 83.59375)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "id": "8omCxmqxOCVz",
        "outputId": "0b43d0a7-1213-4c41-89c4-5107e1429fe3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-6fb2519d6ad8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlemmes_inconnus\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'lemmes_inconnus' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m0NvftzZs5TK"
      },
      "source": [
        "Now we need to find the lemmas, targets and the pairs of target + lemma that are not in train+val sets but are present in dev set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QyvuxcYjtZBd"
      },
      "source": [
        "from itertools import chain\n",
        "\n",
        "lemmes_inconnus = []\n",
        "sens_inconnus = []\n",
        "\n",
        "for lemma in tg_lemmas['dev']:\n",
        "  if lemma not in chain(tg_lemmas['train'], tg_lemmas['val']):\n",
        "    lemmes_inconnus.append(lemma)\n",
        "for sens in label_strs['dev']:\n",
        "  if sens not in chain(label_strs['train'], label_strs['val']):\n",
        "    sens_inconnus.append(sens)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7m33grDnxGSw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52c50aae-4034-4e3b-918d-2454c6908ea3"
      },
      "source": [
        "'The number of unknown lemmas in dev is', len(set(lemmes_inconnus))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('The number of unknown lemmas in dev is', 20)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2z3h5B_ZxsLk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d6d839f-55ea-4e80-cb36-b300c4ed0356"
      },
      "source": [
        "'The number of unknown sens in dev is', len(set(sens_inconnus))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('The number of unknown sens in dev is', 0)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dict_lemmes_train['opposition']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "drO_-QGqQj7z",
        "outputId": "cca5f742-5155-4ac2-c477-98f5d5a71295"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "defaultdict(int, {'FR_Being_in_favor_of': 19, 'Other_sense': 35})"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kLnlSS6tyZi2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "882c442a-c5e3-4a9e-e7a2-3bdfe525418f"
      },
      "source": [
        "unknown_associations = []\n",
        "for lemma in dict_lemmes_dev.keys():\n",
        "  if lemma in dict_lemmes_train:\n",
        "    \n",
        "    associations_possibles_train = dict_lemmes_train[lemma].keys()\n",
        "    associations_possibles_val = dict_lemmes_dev[lemma].keys()\n",
        "    print(lemma,associations_possibles_train,associations_possibles_val)\n",
        "    for association_val in associations_possibles_val:\n",
        "      if association_val not in associations_possibles_train:\n",
        "         unknown_associations.append((lemma,association_val))\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "délibérer dict_keys(['Other_sense']) dict_keys(['FR_Chatting-Discussion'])\n",
            "accepter dict_keys(['Respond_to_proposal', 'FR_Taking_sides', 'FR_Agree_or_refuse_to_act', 'Ratification', 'Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Respond_to_proposal', 'FR_Agree_or_refuse_to_act', 'FR_Taking_sides', 'Other_sense'])\n",
            "faire dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation', 'Other_sense'])\n",
            "demande dict_keys(['FR_Request', 'Other_sense']) dict_keys(['FR_Request', 'Other_sense'])\n",
            "coût dict_keys(['FR_Spending']) dict_keys(['FR_Spending'])\n",
            "s'élever dict_keys(['Other_sense', 'FR_Taking_sides']) dict_keys(['Other_sense'])\n",
            "apprécier dict_keys(['Judgment', 'Other_sense']) dict_keys(['Judgment'])\n",
            "dépenser dict_keys(['FR_Spending']) dict_keys(['Other_sense', 'FR_Spending'])\n",
            "pour dict_keys(['Other_sense', 'FR_Means_for_purpose']) dict_keys(['Other_sense', 'FR_Means_for_purpose'])\n",
            "entraîner dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense', 'Causation'])\n",
            "obliger dict_keys(['FR_Reason', 'Other_sense']) dict_keys(['FR_Reason'])\n",
            "connaissance dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "apprendre dict_keys(['FR_Cognizer_affecting.veracity', 'Other_sense']) dict_keys(['FR_Cognizer_affecting.veracity', 'Other_sense'])\n",
            "répondre dict_keys(['Other_sense', 'Communication_response', 'Response']) dict_keys(['Other_sense', 'Communication_response', 'Response'])\n",
            "animer dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "comme dict_keys(['Other_sense', 'Evidence', 'Causation']) dict_keys(['Other_sense', 'Causation'])\n",
            "parler dict_keys(['FR_Speak_on_topic', 'Other_sense', 'FR_Chatting-Discussion', 'FR_Purpose']) dict_keys(['FR_Speak_on_topic', 'Other_sense', 'FR_Purpose'])\n",
            "réveiller dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['FR_Cause_to_start-Launch_process'])\n",
            "souvenir dict_keys(['FR_Memory-Remembering_experience-Remembering_information', 'Other_sense']) dict_keys(['FR_Memory-Remembering_experience-Remembering_information'])\n",
            "souligner dict_keys(['Convey_importance', 'Evidence']) dict_keys(['Convey_importance', 'Evidence'])\n",
            "tenir dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "naître dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense'])\n",
            "alors dict_keys(['Other_sense', 'Causation', 'Evidence']) dict_keys(['Other_sense', 'Causation', 'Evidence'])\n",
            "opposition dict_keys(['FR_Being_in_favor_of', 'Other_sense']) dict_keys(['Be_in_agreement_on_assessment', 'Other_sense'])\n",
            "accuser dict_keys(['FR_Judgment_communication', 'Other_sense']) dict_keys(['FR_Judgment_communication', 'Other_sense'])\n",
            "décision dict_keys(['FR_Deciding']) dict_keys(['FR_Deciding'])\n",
            "selon dict_keys(['Other_sense', 'FR_Attributed_information']) dict_keys(['FR_Attributed_information', 'Other_sense'])\n",
            "pour_que dict_keys(['Other_sense', 'FR_Means_for_purpose']) dict_keys(['Other_sense', 'FR_Means_for_purpose'])\n",
            "sentir dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "responsable dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense', 'Causation'])\n",
            "céder dict_keys(['Other_sense', 'Commerce_sell']) dict_keys(['Other_sense', 'Commerce_sell'])\n",
            "signe dict_keys(['Other_sense', 'Evidence']) dict_keys(['Other_sense', 'Evidence'])\n",
            "conduire dict_keys(['Other_sense', 'Causation', 'FR_Reason']) dict_keys(['Other_sense', 'FR_Reason'])\n",
            "recouvrer dict_keys(['FR_Getting_money', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "ordre_du_jour dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "décider dict_keys(['FR_Deciding', 'FR_Purpose', 'FR_Cognizer_affecting.to_do', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Deciding'])\n",
            "acheter dict_keys(['Commerce_buy']) dict_keys(['Commerce_buy'])\n",
            "car dict_keys(['Evidence', 'Causation', 'FR_Cause_enunciation', 'Other_sense']) dict_keys(['FR_Cause_enunciation', 'Causation', 'Evidence'])\n",
            "expliquer dict_keys(['FR_Speak_on_topic', 'FR_Statement-manner-noise', 'FR_Attributing_cause', 'Explaining_the_facts']) dict_keys(['FR_Statement-manner-noise', 'FR_Speak_on_topic', 'Explaining_the_facts', 'FR_Attributing_cause'])\n",
            "dès_lors dict_keys(['Causation', 'Evidence', 'Other_sense', 'FR_Cause_enunciation']) dict_keys(['Causation', 'Evidence'])\n",
            "condamnation dict_keys(['FR_Judgment_communication', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "prévoir dict_keys(['Other_sense', 'FR_Expectation']) dict_keys(['Other_sense', 'FR_Expectation'])\n",
            "versement dict_keys(['FR_Giving_money']) dict_keys(['FR_Giving_money'])\n",
            "aussi dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "découvrir dict_keys(['FR_Becoming_aware', 'Other_sense']) dict_keys(['FR_Becoming_aware'])\n",
            "exposé dict_keys(['FR_Speak_on_topic', 'Other_sense']) dict_keys(['FR_Speak_on_topic'])\n",
            "hypothèse dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "éviter dict_keys(['Other_sense', 'Preventing']) dict_keys(['Preventing', 'Other_sense'])\n",
            "reprise dict_keys(['Other_sense', 'Commerce_buy']) dict_keys(['Other_sense', 'Commerce_buy'])\n",
            "prévenir dict_keys(['FR_Telling', 'Preventing']) dict_keys(['Preventing', 'FR_Telling'])\n",
            "constater dict_keys(['FR_Becoming_aware', 'FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise', 'FR_Becoming_aware'])\n",
            "exigence dict_keys(['Other_sense', 'FR_Request']) dict_keys(['FR_Request', 'Other_sense'])\n",
            "témoignage dict_keys(['FR_Speak_on_topic', 'Other_sense']) dict_keys(['FR_Speak_on_topic', 'Other_sense'])\n",
            "faire_état dict_keys(['FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise'])\n",
            "sembler dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "résultat dict_keys(['Other_sense', 'Causation']) dict_keys(['Causation', 'Other_sense'])\n",
            "permettre dict_keys(['Make_possible_to_do', 'FR_Grant_permission-Permitting']) dict_keys(['Make_possible_to_do', 'FR_Grant_permission-Permitting'])\n",
            "porter dict_keys(['Other_sense', 'FR_Support_verb', 'Causation']) dict_keys(['Other_sense', 'FR_Support_verb', 'Causation'])\n",
            "alerter dict_keys(['FR_Telling']) dict_keys(['FR_Telling'])\n",
            "reconnaissance dict_keys(['FR_Statement-manner-noise', 'Other_sense', 'FR_Expressing_decision', 'FR_Taking_sides']) dict_keys(['Other_sense', 'FR_Statement-manner-noise'])\n",
            "créer dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process'])\n",
            "compter dict_keys(['Other_sense', 'FR_Purpose']) dict_keys(['Other_sense', 'FR_Purpose'])\n",
            "évoquer dict_keys(['FR_Speak_on_topic', 'Evoking', 'Adducing']) dict_keys(['FR_Speak_on_topic'])\n",
            "projet dict_keys(['FR_Purpose', 'Other_sense']) dict_keys(['FR_Purpose', 'Other_sense'])\n",
            "jouer dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense', 'FR_Contingency-Objective_influence'])\n",
            "voir dict_keys(['Other_sense', 'FR_Becoming_aware', 'Evidence']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion', 'FR_Reason'])\n",
            "raconter dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "envisager dict_keys(['FR_Purpose', 'Other_sense', 'FR_Expectation', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Purpose'])\n",
            "acquisition dict_keys(['Commerce_buy']) dict_keys(['Commerce_buy'])\n",
            "réaliser dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "s'engager dict_keys(['FR_Commitment.to_do', 'Other_sense']) dict_keys(['FR_Commitment.to_do', 'Other_sense'])\n",
            "préciser dict_keys(['FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise'])\n",
            "conseil dict_keys(['Other_sense', 'FR_Attempt_suasion.to_do']) dict_keys(['Other_sense', 'FR_Attempt_suasion.to_do'])\n",
            "création dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "assurance dict_keys(['Other_sense', 'FR_Commitment.to_do']) dict_keys(['Other_sense'])\n",
            "contrat dict_keys(['FR_Having_commercial_agreement', 'Be_in_agreement_on_action']) dict_keys(['FR_Having_commercial_agreement', 'Be_in_agreement_on_action'])\n",
            "inviter dict_keys(['FR_Attempt_suasion.to_do', 'Other_sense', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "prêt dict_keys(['Willingness', 'Other_sense']) dict_keys(['Willingness', 'Other_sense'])\n",
            "convier dict_keys(['FR_Attempt_suasion.to_do', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['Other_sense'])\n",
            "afin_de dict_keys(['FR_Means_for_purpose']) dict_keys(['FR_Means_for_purpose'])\n",
            "aboutir dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense', 'Causation'])\n",
            "recommandation dict_keys(['FR_Attempt_suasion.to_do', 'Other_sense']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "condition dict_keys(['Other_sense', 'Make_possible_to_do']) dict_keys(['Other_sense', 'Make_possible_to_do'])\n",
            "dépendre dict_keys(['FR_Contingency-Objective_influence', 'Other_sense']) dict_keys(['FR_Contingency-Objective_influence'])\n",
            "motif dict_keys(['FR_Reason', 'Other_sense', 'Causation']) dict_keys(['FR_Reason', 'Other_sense'])\n",
            "subir dict_keys(['FR_Support_verb', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Support_verb', 'FR_Contingency-Objective_influence'])\n",
            "jusqu'à dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "résumé dict_keys(['Summarizing']) dict_keys(['Summarizing'])\n",
            "produit dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "atteindre dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Contingency-Objective_influence', 'Other_sense'])\n",
            "faire_l'objet dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "recommander dict_keys(['FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "à_raison_de dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "effet dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "réaction dict_keys(['Response', 'Other_sense']) dict_keys(['Response', 'Other_sense'])\n",
            "mener dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense', 'Causation'])\n",
            "exiger dict_keys(['Other_sense', 'FR_Request']) dict_keys(['Other_sense', 'FR_Request'])\n",
            "source dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense'])\n",
            "rapporter dict_keys(['FR_Statement-manner-noise', 'FR_Cause_earning', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "montrer dict_keys(['Other_sense', 'Evidence', 'FR_Proving']) dict_keys(['Evidence', 'Other_sense', 'FR_Proving'])\n",
            "induire dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation'])\n",
            "réponse dict_keys(['Communication_response', 'Response']) dict_keys(['Response', 'Communication_response'])\n",
            "ouvrir dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process'])\n",
            "considérer dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "se_justifier dict_keys(['Other_sense', 'FR_Justifying']) dict_keys(['Other_sense'])\n",
            "point_de_vue dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "distribuer dict_keys(['Other_sense', 'Carry_goods']) dict_keys(['Other_sense', 'Carry_goods'])\n",
            "distribution dict_keys(['Other_sense', 'Carry_goods']) dict_keys(['Other_sense', 'Carry_goods'])\n",
            "ordre dict_keys(['FR_Request', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "fonction dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense', 'FR_Contingency-Objective_influence'])\n",
            "s'attendre dict_keys(['FR_Expectation']) dict_keys(['FR_Expectation'])\n",
            "autorisation dict_keys(['FR_Grant_permission-Permitting']) dict_keys(['FR_Grant_permission-Permitting'])\n",
            "marché dict_keys(['Other_sense', 'FR_Having_commercial_agreement']) dict_keys(['Other_sense', 'FR_Having_commercial_agreement'])\n",
            "détailler dict_keys(['FR_Speak_on_topic', 'Other_sense']) dict_keys(['FR_Speak_on_topic'])\n",
            "convenir dict_keys(['Other_sense', 'FR_Make_agreement_on_assessment', 'FR_Awareness-Certainty-Opinion', 'Make_agreement_on_action', 'FR_Statement-manner-noise']) dict_keys(['FR_Make_agreement_on_assessment', 'Make_agreement_on_action', 'Other_sense'])\n",
            "autoriser dict_keys(['FR_Grant_permission-Permitting', 'Make_possible_to_do']) dict_keys(['FR_Grant_permission-Permitting'])\n",
            "indiquer dict_keys(['Evidence', 'FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise', 'Evidence'])\n",
            "vue dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "mention dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "connaître dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "empêcher dict_keys(['Other_sense', 'Preventing']) dict_keys(['Preventing'])\n",
            "contact dict_keys(['FR_Contacting', 'FR_Chatting-Discussion', 'Other_sense']) dict_keys(['FR_Contacting', 'FR_Chatting-Discussion'])\n",
            "impliquer dict_keys(['Other_sense', 'Causation', 'Evidence']) dict_keys(['Other_sense'])\n",
            "observer dict_keys(['FR_Becoming_aware', 'FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Becoming_aware', 'FR_Statement-manner-noise', 'Other_sense'])\n",
            "pourquoi dict_keys(['Causation', 'Evidence']) dict_keys(['Causation'])\n",
            "approuver dict_keys(['Ratification', 'Judgment', 'FR_Judgment_communication']) dict_keys(['Ratification'])\n",
            "estimer dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense', 'FR_Statement-manner-noise']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense', 'FR_Statement-manner-noise'])\n",
            "répétition dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "répéter dict_keys(['Other_sense', 'FR_Statement-manner-noise']) dict_keys(['Other_sense', 'FR_Statement-manner-noise'])\n",
            "incidence dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense', 'FR_Contingency-Objective_influence'])\n",
            "puisque dict_keys(['Evidence', 'FR_Reason']) dict_keys(['Evidence'])\n",
            "conseiller dict_keys(['Other_sense', 'FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do', 'Other_sense'])\n",
            "informer dict_keys(['FR_Telling']) dict_keys(['FR_Telling'])\n",
            "symptôme dict_keys(['Evidence']) dict_keys(['Evidence'])\n",
            "suggérer dict_keys(['FR_Attempt_suasion.to_do', 'FR_Statement-manner-noise', 'Evidence']) dict_keys(['Evidence', 'FR_Attempt_suasion.to_do', 'FR_Statement-manner-noise'])\n",
            "facteur dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation', 'Other_sense'])\n",
            "révéler dict_keys(['Evidence', 'Reveal_secret', 'Other_sense']) dict_keys(['Evidence', 'Reveal_secret', 'Other_sense'])\n",
            "perte dict_keys(['Other_sense', 'Earnings_and_losses']) dict_keys(['Other_sense', 'Earnings_and_losses'])\n",
            "concevoir dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "appeler dict_keys(['FR_Hail', 'Referring_by_name', 'FR_Attempt_suasion.to_do', 'Name_conferral', 'Other_sense', 'FR_Contacting']) dict_keys(['Referring_by_name', 'Other_sense', 'FR_Attempt_suasion.to_do'])\n",
            "certain dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "discuter dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "demander dict_keys(['FR_Request', 'Questioning']) dict_keys(['FR_Request', 'Questioning'])\n",
            "affecter dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Contingency-Objective_influence', 'Other_sense'])\n",
            "savoir dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "initier dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense'])\n",
            "vendre dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell'])\n",
            "commerce dict_keys(['FR_Commerce_scenario', 'Other_sense']) dict_keys(['FR_Commerce_scenario', 'Other_sense'])\n",
            "remercier dict_keys(['Judgment_direct_address', 'Other_sense']) dict_keys(['Judgment_direct_address', 'Other_sense'])\n",
            "imputable dict_keys(['FR_Attributing_cause']) dict_keys(['FR_Attributing_cause'])\n",
            "objectif dict_keys(['FR_Purpose']) dict_keys(['FR_Purpose'])\n",
            "en_vue_de dict_keys(['Other_sense', 'FR_Means_for_purpose']) dict_keys(['FR_Means_for_purpose'])\n",
            "contre dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "en_effet dict_keys(['Evidence', 'FR_Cause_enunciation', 'Causation', 'Other_sense']) dict_keys(['Other_sense', 'Evidence', 'Causation'])\n",
            "raison dict_keys(['FR_Reason', 'Other_sense', 'Evidence', 'Causation']) dict_keys(['FR_Reason', 'Other_sense', 'Causation'])\n",
            "paraître dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "donc dict_keys(['Evidence', 'Causation', 'Other_sense']) dict_keys(['Evidence', 'Other_sense', 'Causation'])\n",
            "avancer dict_keys(['Other_sense', 'FR_Statement-manner-noise']) dict_keys(['Other_sense', 'FR_Statement-manner-noise'])\n",
            "prendre_position dict_keys(['FR_Expressing_side', 'Other_sense']) dict_keys(['FR_Expressing_side', 'Other_sense'])\n",
            "payer dict_keys(['Commerce_pay', 'Other_sense']) dict_keys(['Commerce_pay'])\n",
            "apporter dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense', 'Causation'])\n",
            "assurer dict_keys(['FR_Commitment.veracity', 'Other_sense']) dict_keys(['Other_sense', 'FR_Commitment.veracity'])\n",
            "d'où dict_keys(['Causation', 'Evidence']) dict_keys(['Evidence', 'Causation'])\n",
            "rappeler dict_keys(['FR_Statement-manner-noise', 'Evoking', 'Other_sense']) dict_keys(['FR_Statement-manner-noise', 'Evoking'])\n",
            "grâce_à dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation', 'Other_sense'])\n",
            "engagement dict_keys(['FR_Commitment.to_do', 'Other_sense']) dict_keys(['FR_Commitment.to_do', 'Other_sense'])\n",
            "cause dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "afin_que dict_keys(['FR_Means_for_purpose']) dict_keys(['FR_Means_for_purpose'])\n",
            "penser dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense', 'Regard']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "conscience dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "soutenir dict_keys(['Other_sense', 'FR_Contingency-Objective_influence', 'FR_Being_in_favor_of', 'FR_Statement-manner-noise']) dict_keys(['FR_Being_in_favor_of', 'FR_Statement-manner-noise'])\n",
            "parce_que dict_keys(['Causation', 'Evidence']) dict_keys(['Evidence', 'Causation'])\n",
            "oublier dict_keys(['Remembering_to_do', 'FR_Memory-Remembering_experience-Remembering_information']) dict_keys(['FR_Memory-Remembering_experience-Remembering_information', 'Remembering_to_do'])\n",
            "dire dict_keys(['FR_Statement-manner-noise', 'FR_Being_named', 'Other_sense', 'Evidence', 'FR_Encoding', 'FR_Make_agreement_on_assessment']) dict_keys(['FR_Statement-manner-noise', 'FR_Being_named', 'FR_Encoding', 'Other_sense'])\n",
            "fonder dict_keys(['Other_sense', 'FR_Reason']) dict_keys(['Other_sense'])\n",
            "contrôle dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "importation dict_keys(['Importing']) dict_keys(['Importing'])\n",
            "refuser dict_keys(['FR_Agree_or_refuse_to_act', 'Other_sense', 'FR_Taking_sides', 'Respond_to_proposal', 'FR_Grant_permission-Permitting']) dict_keys(['Respond_to_proposal', 'FR_Agree_or_refuse_to_act', 'FR_Taking_sides'])\n",
            "ainsi dict_keys(['Other_sense', 'Causation']) dict_keys(['Causation', 'Other_sense'])\n",
            "déclaration dict_keys(['FR_Statement-manner-noise', 'FR_Expressing_decision']) dict_keys(['FR_Statement-manner-noise'])\n",
            "à_la_demande_de dict_keys(['Response']) dict_keys(['Response'])\n",
            "débuter dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "confession dict_keys(['Reveal_secret']) dict_keys(['Other_sense'])\n",
            "solliciter dict_keys(['FR_Request', 'Other_sense']) dict_keys(['FR_Request'])\n",
            "remettre dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "rejeter dict_keys(['FR_Taking_sides', 'Respond_to_proposal', 'Other_sense']) dict_keys(['Respond_to_proposal', 'FR_Taking_sides'])\n",
            "s'adresser dict_keys(['FR_Speak_on_topic', 'Other_sense']) dict_keys(['FR_Speak_on_topic'])\n",
            "but dict_keys(['FR_Purpose']) dict_keys(['FR_Purpose'])\n",
            "admettre dict_keys(['FR_Awareness-Certainty-Opinion', 'FR_Statement-manner-noise', 'Other_sense', 'FR_Being_in_favor_of', 'FR_Agree_or_refuse_to_act']) dict_keys(['FR_Awareness-Certainty-Opinion', 'FR_Statement-manner-noise', 'Other_sense', 'FR_Being_in_favor_of'])\n",
            "rapport dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['Other_sense', 'FR_Statement-manner-noise'])\n",
            "s'exprimer dict_keys(['FR_Speak_on_topic', 'FR_Expressing_side', 'Other_sense']) dict_keys(['Other_sense', 'FR_Speak_on_topic'])\n",
            "règlement dict_keys(['Other_sense', 'Commerce_pay']) dict_keys(['Other_sense'])\n",
            "c'est_pourquoi dict_keys(['Evidence', 'Causation']) dict_keys(['Evidence'])\n",
            "laisser dict_keys(['Other_sense', 'Preventing', 'Causation']) dict_keys(['Causation', 'Preventing', 'Other_sense'])\n",
            "lancer dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense']) dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense'])\n",
            "appel dict_keys(['FR_Contacting', 'Other_sense', 'FR_Attempt_suasion.to_do', 'FR_Hail']) dict_keys(['FR_Attempt_suasion.to_do', 'Other_sense'])\n",
            "accord dict_keys(['Be_in_agreement_on_assessment', 'Be_in_agreement_on_action']) dict_keys(['Be_in_agreement_on_action', 'FR_Being_in_favor_of'])\n",
            "arrangement dict_keys(['Be_in_agreement_on_action']) dict_keys(['Be_in_agreement_on_action'])\n",
            "en_raison_de dict_keys(['Evidence', 'Causation']) dict_keys(['Evidence', 'Causation', 'Other_sense'])\n",
            "se_souvenir dict_keys(['FR_Memory-Remembering_experience-Remembering_information']) dict_keys(['FR_Memory-Remembering_experience-Remembering_information'])\n",
            "annoncer dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "gain dict_keys(['Other_sense', 'Earnings_and_losses']) dict_keys(['Earnings_and_losses', 'Other_sense'])\n",
            "engendrer dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense']) dict_keys(['FR_Cause_to_start-Launch_process'])\n",
            "résolution dict_keys(['Other_sense', 'FR_Deciding']) dict_keys(['Other_sense'])\n",
            "relever dict_keys(['Other_sense', 'FR_Becoming_aware', 'Convey_importance']) dict_keys(['Other_sense', 'Convey_importance', 'FR_Becoming_aware'])\n",
            "résulter dict_keys(['Other_sense', 'Causation']) dict_keys(['Causation', 'Other_sense'])\n",
            "produire dict_keys(['Other_sense', 'Causation']) dict_keys(['Causation', 'Other_sense'])\n",
            "dépense dict_keys(['FR_Spending', 'Other_sense']) dict_keys(['FR_Spending'])\n",
            "aborder dict_keys(['Other_sense', 'FR_Speak_on_topic']) dict_keys(['FR_Speak_on_topic'])\n",
            "déceler dict_keys(['FR_Becoming_aware']) dict_keys(['FR_Becoming_aware'])\n",
            "conclure dict_keys(['Coming_to_believe', 'FR_Support_verb', 'FR_Statement-manner-noise', 'Other_sense']) dict_keys(['Coming_to_believe', 'FR_Statement-manner-noise', 'FR_Support_verb', 'Other_sense'])\n",
            "en_vertu_de dict_keys(['FR_Reason', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "conclusion dict_keys(['Coming_to_believe', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "avouer dict_keys(['Reveal_secret']) dict_keys(['Reveal_secret'])\n",
            "sentiment dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "propos dict_keys(['FR_Speak_on_topic']) dict_keys(['FR_Speak_on_topic'])\n",
            "signaler dict_keys(['FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise'])\n",
            "regretter dict_keys(['FR_Taking_sides', 'Other_sense']) dict_keys(['FR_Taking_sides'])\n",
            "condamner dict_keys(['Other_sense', 'FR_Judgment_communication']) dict_keys(['FR_Judgment_communication', 'Other_sense'])\n",
            "argumenter dict_keys(['FR_Attempt_supported_suasion']) dict_keys(['FR_Attempt_supported_suasion'])\n",
            "opinion dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "amener dict_keys(['FR_Reason', 'Other_sense', 'Causation']) dict_keys(['FR_Reason', 'Other_sense'])\n",
            "convaincre dict_keys(['FR_Awareness-Certainty-Opinion', 'FR_Cognizer_affecting.veracity', 'FR_Cognizer_affecting.to_do']) dict_keys(['FR_Cognizer_affecting.veracity', 'FR_Cognizer_affecting.to_do', 'FR_Awareness-Certainty-Opinion'])\n",
            "marque dict_keys(['Other_sense', 'Evidence']) dict_keys(['Other_sense'])\n",
            "percevoir dict_keys(['FR_Getting_money', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion', 'FR_Getting_money'])\n",
            "aveu dict_keys(['Reveal_secret']) dict_keys(['Reveal_secret'])\n",
            "déclarer dict_keys(['FR_Statement-manner-noise', 'FR_Expressing_decision']) dict_keys(['FR_Expressing_decision', 'FR_Statement-manner-noise'])\n",
            "menacer dict_keys(['FR_Commitment.to_do', 'Other_sense']) dict_keys(['FR_Commitment.to_do', 'Other_sense'])\n",
            "nommer dict_keys(['Name_conferral', 'Appointing', 'Referring_by_name']) dict_keys(['Appointing'])\n",
            "défense dict_keys(['Other_sense', 'FR_Justifying']) dict_keys(['Other_sense'])\n",
            "contraindre dict_keys(['FR_Reason', 'Causation', 'Other_sense']) dict_keys(['FR_Reason'])\n",
            "révélation dict_keys(['Reveal_secret', 'Other_sense']) dict_keys(['Reveal_secret'])\n",
            "suite dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation', 'Other_sense'])\n",
            "confier dict_keys(['Reveal_secret', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "ajouter dict_keys(['Other_sense', 'FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise', 'Other_sense'])\n",
            "doute dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "vente dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell'])\n",
            "verser dict_keys(['FR_Giving_money', 'Other_sense']) dict_keys(['FR_Giving_money'])\n",
            "à_des_fins_de dict_keys(['FR_Means_for_purpose']) dict_keys(['FR_Means_for_purpose'])\n",
            "opposé dict_keys(['Other_sense']) dict_keys(['FR_Being_in_favor_of', 'Be_in_agreement_on_assessment'])\n",
            "rembourser dict_keys(['FR_Reimbursement', 'Repayment']) dict_keys(['FR_Reimbursement'])\n",
            "interdire dict_keys(['FR_Grant_permission-Permitting', 'Preventing']) dict_keys(['FR_Grant_permission-Permitting'])\n",
            "achat dict_keys(['Commerce_buy', 'Other_sense']) dict_keys(['Commerce_buy'])\n",
            "écrire dict_keys(['FR_Statement-manner-noise', 'Text_creation', 'FR_Contacting', 'Other_sense']) dict_keys(['FR_Statement-manner-noise', 'FR_Contacting', 'Text_creation'])\n",
            "refus dict_keys(['FR_Agree_or_refuse_to_act', 'Respond_to_proposal', 'FR_Taking_sides', 'FR_Grant_permission-Permitting']) dict_keys(['FR_Agree_or_refuse_to_act'])\n",
            "trouver dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "plaidoirie dict_keys(['FR_Attempt_supported_suasion']) dict_keys(['FR_Attempt_suasion.legitimacy'])\n",
            "confirmer dict_keys(['Evidence', 'FR_Expressing_truth_of_proposition', 'Ratification', 'Other_sense']) dict_keys(['FR_Expressing_truth_of_proposition', 'Evidence', 'Ratification', 'Other_sense'])\n",
            "paiement dict_keys(['Commerce_pay']) dict_keys(['Commerce_pay'])\n",
            "désignation dict_keys(['Appointing', 'Other_sense']) dict_keys(['Appointing'])\n",
            "classer dict_keys(['Categorization', 'Other_sense']) dict_keys(['Other_sense', 'Categorization'])\n",
            "rendre dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense', 'Causation'])\n",
            "rétorquer dict_keys(['Communication_response']) dict_keys(['Communication_response'])\n",
            "comprendre dict_keys(['Other_sense', 'FR_Becoming_aware']) dict_keys(['Other_sense', 'FR_Becoming_aware'])\n",
            "influence dict_keys(['Other_sense', 'FR_Contingency-Objective_influence', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['FR_Influence_of_event_on_cognizer-Subjective_influence'])\n",
            "rémunérer dict_keys(['Commerce_pay', 'Commerce_buy']) dict_keys(['Commerce_pay', 'Commerce_buy'])\n",
            "prélever dict_keys(['Other_sense', 'FR_Getting_money']) dict_keys(['FR_Getting_money'])\n",
            "reverser dict_keys(['FR_Giving_money']) dict_keys(['FR_Giving_money'])\n",
            "crier dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "affirmer dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "prier dict_keys(['Other_sense', 'FR_Request']) dict_keys(['FR_Request'])\n",
            "déclencher dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense']) dict_keys(['FR_Cause_to_start-Launch_process'])\n",
            "à_l'appel_de dict_keys(['FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "dénoncer dict_keys(['FR_Judgment_communication', 'Other_sense']) dict_keys(['FR_Judgment_communication'])\n",
            "protester dict_keys(['Complaining']) dict_keys(['Complaining'])\n",
            "réclamer dict_keys(['Other_sense', 'FR_Request']) dict_keys(['FR_Request'])\n",
            "prévision dict_keys(['FR_Expectation', 'Other_sense']) dict_keys(['FR_Expectation'])\n",
            "contrôler dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense', 'FR_Contingency-Objective_influence'])\n",
            "origine dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense', 'Causation'])\n",
            "divulguer dict_keys(['Reveal_secret']) dict_keys(['Reveal_secret'])\n",
            "noter dict_keys(['FR_Becoming_aware', 'FR_Statement-manner-noise']) dict_keys(['FR_Statement-manner-noise'])\n",
            "citer dict_keys(['FR_Statement-manner-noise', 'FR_Quoting', 'Adducing', 'Other_sense']) dict_keys(['FR_Quoting', 'FR_Statement-manner-noise'])\n",
            "réagir dict_keys(['Response']) dict_keys(['Response'])\n",
            "reprendre dict_keys(['Other_sense', 'Commerce_buy']) dict_keys(['Other_sense', 'Commerce_buy'])\n",
            "opposer dict_keys(['Communication_response', 'FR_Being_in_favor_of', 'Other_sense']) dict_keys(['FR_Being_in_favor_of', 'Other_sense'])\n",
            "approbation dict_keys(['Ratification']) dict_keys(['Ratification'])\n",
            "s'appeler dict_keys(['FR_Being_named']) dict_keys(['FR_Being_named'])\n",
            "déduire dict_keys(['Other_sense', 'Coming_to_believe']) dict_keys(['Other_sense', 'Coming_to_believe'])\n",
            "soutien dict_keys(['Other_sense', 'FR_Being_in_favor_of']) dict_keys(['Other_sense', 'FR_Being_in_favor_of'])\n",
            "cotisation dict_keys(['Commerce_pay']) dict_keys(['Commerce_pay'])\n",
            "devoir dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense'])\n",
            "coûter dict_keys(['FR_Spending']) dict_keys(['FR_Spending'])\n",
            "de_l'ordre_de dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "perdre dict_keys(['Other_sense', 'Earnings_and_losses']) dict_keys(['Earnings_and_losses', 'Other_sense'])\n",
            "à_cause_de dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "privatisation dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell'])\n",
            "retombée dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation'])\n",
            "acquérir dict_keys(['Other_sense', 'Commerce_buy']) dict_keys(['Commerce_buy', 'Other_sense'])\n",
            "gagner dict_keys(['Other_sense', 'Earnings_and_losses']) dict_keys(['Earnings_and_losses', 'Other_sense'])\n",
            "importer dict_keys(['Other_sense', 'Importing']) dict_keys(['Importing'])\n",
            "commercialisation dict_keys(['Carry_goods']) dict_keys(['Carry_goods'])\n",
            "promettre dict_keys(['FR_Commitment.to_do', 'Other_sense']) dict_keys(['FR_Commitment.to_do'])\n",
            "marquer dict_keys(['Other_sense', 'Evidence', 'FR_Contingency-Objective_influence']) dict_keys(['Evidence', 'Other_sense'])\n",
            "soupçonner dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "engager dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense'])\n",
            "en_coûter dict_keys(['FR_Spending']) dict_keys(['FR_Spending'])\n",
            "consentir dict_keys(['FR_Agree_or_refuse_to_act_in_favor_of', 'FR_Agree_or_refuse_to_act']) dict_keys(['FR_Agree_or_refuse_to_act_in_favor_of'])\n",
            "rémunération dict_keys(['Commerce_pay', 'Commerce_buy']) dict_keys(['Commerce_buy', 'Commerce_pay'])\n",
            "acquitter dict_keys(['Other_sense', 'Commerce_pay']) dict_keys(['Commerce_pay'])\n",
            "indice dict_keys(['Evidence', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "remboursement dict_keys(['FR_Reimbursement', 'Repayment']) dict_keys(['FR_Reimbursement', 'Repayment'])\n",
            "exportation dict_keys(['Exporting']) dict_keys(['Exporting'])\n",
            "résoudre dict_keys(['Other_sense', 'FR_Purpose']) dict_keys(['FR_Purpose'])\n",
            "enjoindre dict_keys(['FR_Request']) dict_keys(['FR_Request'])\n",
            "faire_preuve dict_keys(['Other_sense']) dict_keys(['Other_sense', 'FR_Proving'])\n",
            "commercialiser dict_keys(['Carry_goods']) dict_keys(['Carry_goods'])\n",
            "entendre dict_keys(['Other_sense', 'FR_Purpose']) dict_keys(['FR_Purpose', 'Other_sense'])\n",
            "acceptation dict_keys(['FR_Taking_sides']) dict_keys(['Ratification'])\n",
            "faire_remarquer dict_keys(['Other_sense', 'FR_Telling']) dict_keys(['FR_Telling'])\n",
            "transaction dict_keys(['FR_Commercial_transaction']) dict_keys(['FR_Commercial_transaction'])\n",
            "cession dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell'])\n",
            "racheter dict_keys(['Commerce_buy', 'Other_sense']) dict_keys(['Commerce_buy'])\n",
            "interrogation dict_keys(['Questioning']) dict_keys(['Questioning'])\n",
            "fruit dict_keys(['Other_sense', 'Causation']) dict_keys(['Causation'])\n",
            "encourager dict_keys(['Other_sense', 'FR_Influence_of_event_on_cognizer-Subjective_influence', 'FR_Attempt_suasion.to_do', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Attempt_suasion.to_do', 'FR_Influence_of_event_on_cognizer-Subjective_influence'])\n",
            "idée dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "contestation dict_keys(['FR_Taking_sides']) dict_keys(['FR_Taking_sides'])\n",
            "tabler dict_keys(['FR_Reliance_on_expectation']) dict_keys(['FR_Reliance_on_expectation'])\n",
            "plaider dict_keys(['Other_sense', 'FR_Attempt_suasion.legitimacy', 'FR_Deserving', 'FR_Justifying', 'FR_Attempt_supported_suasion']) dict_keys(['FR_Attempt_suasion.legitimacy'])\n",
            "intention dict_keys(['Other_sense', 'FR_Purpose']) dict_keys(['Other_sense', 'FR_Purpose'])\n",
            "imputer dict_keys(['FR_Attributing_cause']) dict_keys(['FR_Attributing_cause'])\n",
            "provoquer dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "inciter dict_keys(['FR_Influence_of_event_on_cognizer-Subjective_influence', 'FR_Attempt_suasion.to_do']) dict_keys(['FR_Influence_of_event_on_cognizer-Subjective_influence'])\n",
            "critiquer dict_keys(['FR_Judgment_communication']) dict_keys(['FR_Judgment_communication'])\n",
            "reconnaître dict_keys(['FR_Statement-manner-noise', 'FR_Expressing_decision', 'FR_Taking_sides', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "préjugé dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "écrit dict_keys(['Text_creation', 'Other_sense']) dict_keys(['Text_creation'])\n",
            "songer dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "disposer dict_keys(['Other_sense', 'Willingness']) dict_keys(['Other_sense'])\n",
            "attirer dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "incitation dict_keys(['FR_Attempt_suasion.to_do', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "répercussion dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "commencer dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense'])\n",
            "interpréter dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "c'est_que dict_keys(['Explaining_the_facts']) dict_keys(['Explaining_the_facts'])\n",
            "témoigner dict_keys(['FR_Speak_on_topic', 'Evidence', 'Other_sense']) dict_keys(['Evidence'])\n",
            "argument dict_keys(['FR_Attempt_supported_suasion']) dict_keys(['FR_Attempt_supported_suasion'])\n",
            "nier dict_keys(['FR_Expressing_truth_of_proposition']) dict_keys(['FR_Expressing_truth_of_proposition'])\n",
            "juger dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion', 'Regard']) dict_keys(['Regard', 'FR_Awareness-Certainty-Opinion'])\n",
            "ignorance dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "communication dict_keys(['Other_sense', 'FR_Telling', 'FR_Speak_on_topic', 'FR_Chatting-Discussion']) dict_keys(['Other_sense', 'FR_Speak_on_topic'])\n",
            "ignorer dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "prouver dict_keys(['Evidence', 'FR_Proving']) dict_keys(['Evidence', 'FR_Proving'])\n",
            "nomination dict_keys(['Appointing']) dict_keys(['Appointing'])\n",
            "résumer dict_keys(['Summarizing', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "préconiser dict_keys(['FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "entamer dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "se_résoudre dict_keys(['FR_Deciding', 'Other_sense']) dict_keys(['FR_Deciding'])\n",
            "débattre dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "justifier dict_keys(['FR_Deserving', 'Other_sense', 'FR_Justifying', 'FR_Attributing_cause']) dict_keys(['FR_Justifying', 'Other_sense'])\n",
            "débat dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "c'est_-_à_-_dire dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "interroger dict_keys(['Questioning']) dict_keys(['Questioning'])\n",
            "critique dict_keys(['Other_sense', 'FR_Judgment_communication']) dict_keys(['FR_Judgment_communication'])\n",
            "inspirer dict_keys(['Other_sense', 'FR_Reason', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['FR_Reason'])\n",
            "attribuer dict_keys(['Other_sense', 'FR_Attributing_cause']) dict_keys(['FR_Attributing_cause'])\n",
            "querelle dict_keys(['Quarreling']) dict_keys(['Quarreling'])\n",
            "prévisible dict_keys(['FR_Expectation']) dict_keys(['FR_Expectation'])\n",
            "toucher dict_keys(['Other_sense', 'FR_Contingency-Objective_influence', 'FR_Getting_money']) dict_keys(['FR_Contingency-Objective_influence', 'Other_sense', 'FR_Getting_money'])\n",
            "manifester dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "privatiser dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell'])\n",
            "manifestation dict_keys(['Other_sense', 'Evidence']) dict_keys(['Evidence', 'Other_sense'])\n",
            "interprétation dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "imprévu dict_keys(['FR_Expectation']) dict_keys(['FR_Expectation'])\n",
            "assuré dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "croire dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense', 'FR_Being_in_favor_of']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "liquider dict_keys(['Commerce_sell']) dict_keys(['Commerce_sell', 'Other_sense'])\n",
            "attester dict_keys(['Evidence']) dict_keys(['Evidence'])\n",
            "vanter dict_keys(['FR_Judgment_communication']) dict_keys(['FR_Judgment_communication'])\n",
            "commenter dict_keys(['Other_sense', 'FR_Speak_on_topic']) dict_keys(['Other_sense', 'FR_Speak_on_topic'])\n",
            "ordonner dict_keys(['FR_Request']) dict_keys(['FR_Request'])\n",
            "repousser dict_keys(['Other_sense', 'Respond_to_proposal']) dict_keys(['Other_sense'])\n",
            "faute_de dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation'])\n",
            "discussion dict_keys(['FR_Speak_on_topic', 'FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "suspecter dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "reprocher dict_keys(['Judgment_direct_address']) dict_keys(['Judgment_direct_address'])\n",
            "contrainte dict_keys(['FR_Reason', 'Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['Other_sense'])\n",
            "se_féliciter dict_keys(['FR_Taking_sides']) dict_keys(['FR_Taking_sides'])\n",
            "déplorer dict_keys(['FR_Taking_sides']) dict_keys(['FR_Taking_sides'])\n",
            "attente dict_keys(['Other_sense', 'FR_Expectation']) dict_keys(['Other_sense'])\n",
            "preuve dict_keys(['Evidence', 'FR_Proving']) dict_keys(['Evidence'])\n",
            "démentir dict_keys(['FR_Expressing_truth_of_proposition']) dict_keys(['FR_Expressing_truth_of_proposition'])\n",
            "amorcer dict_keys(['FR_Cause_to_start-Launch_process', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "presser dict_keys(['Other_sense', 'FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "menace dict_keys(['Other_sense', 'FR_Commitment.to_do']) dict_keys(['Other_sense'])\n",
            "pronostiquer dict_keys(['Predicting']) dict_keys(['Predicting'])\n",
            "anticiper dict_keys(['Other_sense', 'FR_Expectation']) dict_keys(['Other_sense'])\n",
            "cotiser dict_keys(['Commerce_pay']) dict_keys(['Commerce_pay'])\n",
            "découverte dict_keys(['FR_Becoming_aware']) dict_keys(['FR_Becoming_aware'])\n",
            "classement dict_keys(['Categorization', 'Other_sense']) dict_keys(['Categorization'])\n",
            "solde dict_keys(['Other_sense', 'Commerce_sell', 'Earnings_and_losses']) dict_keys(['Commerce_sell', 'Other_sense'])\n",
            "atténuer dict_keys(['FR_Contingency-Objective_influence', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "infliger dict_keys(['Other_sense', 'Causation']) dict_keys(['Other_sense'])\n",
            "démontrer dict_keys(['FR_Proving', 'Other_sense', 'Evidence']) dict_keys(['FR_Proving', 'Evidence'])\n",
            "illustrer dict_keys(['Other_sense', 'Evidence']) dict_keys(['Evidence'])\n",
            "contester dict_keys(['FR_Expressing_truth_of_proposition', 'FR_Taking_sides', 'Other_sense']) dict_keys(['FR_Taking_sides'])\n",
            "rachat dict_keys(['Commerce_buy', 'Commerce_sell']) dict_keys(['Commerce_buy'])\n",
            "arguer dict_keys(['FR_Justifying', 'FR_Attempt_suasion.veracity']) dict_keys(['FR_Justifying'])\n",
            "motiver dict_keys(['FR_Reason', 'Other_sense', 'FR_Influence_of_event_on_cognizer-Subjective_influence']) dict_keys(['FR_Reason'])\n",
            "lancement dict_keys(['Other_sense', 'FR_Cause_to_start-Launch_process']) dict_keys(['Other_sense'])\n",
            "notifier dict_keys(['FR_Telling']) dict_keys(['FR_Telling'])\n",
            "se_prononcer dict_keys(['FR_Expressing_side']) dict_keys(['FR_Expressing_side'])\n",
            "faire_suite dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "infléchir dict_keys(['FR_Contingency-Objective_influence']) dict_keys(['FR_Contingency-Objective_influence'])\n",
            "se_plaindre dict_keys(['Complaining']) dict_keys(['Complaining'])\n",
            "alliance dict_keys(['Be_in_agreement_on_action', 'Other_sense']) dict_keys(['Other_sense', 'Be_in_agreement_on_action'])\n",
            "renoncer dict_keys(['FR_Renunciation']) dict_keys(['FR_Renunciation'])\n",
            "favorable dict_keys(['FR_Being_in_favor_of', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "se_défendre dict_keys(['FR_Expressing_truth_of_proposition']) dict_keys(['FR_Expressing_truth_of_proposition'])\n",
            "forcer dict_keys(['FR_Reason', 'Other_sense']) dict_keys(['FR_Reason'])\n",
            "dit dict_keys(['FR_Being_named', 'FR_Statement-manner-noise']) dict_keys(['FR_Being_named'])\n",
            "soulever dict_keys(['FR_Encoding', 'Other_sense', 'FR_Reason']) dict_keys(['FR_Encoding'])\n",
            "rejet dict_keys(['FR_Taking_sides', 'Respond_to_proposal']) dict_keys(['Respond_to_proposal'])\n",
            "ce_être_-_à_-_dire dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "à_la_suite_de dict_keys(['Causation', 'Other_sense']) dict_keys(['Causation', 'Other_sense'])\n",
            "effectivement dict_keys(['Other_sense', 'Evidence']) dict_keys(['Other_sense'])\n",
            "par_voie_de_conséquence dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "entériner dict_keys(['Evidence', 'Ratification']) dict_keys(['Ratification'])\n",
            "avis dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "tractation dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "pourparler dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "en_appeler dict_keys(['FR_Attempt_suasion.to_do']) dict_keys(['FR_Attempt_suasion.to_do'])\n",
            "s'accorder dict_keys(['Be_in_agreement_on_assessment', 'Other_sense']) dict_keys(['Be_in_agreement_on_assessment'])\n",
            "souscrire dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "promesse dict_keys(['FR_Commitment.to_do', 'Other_sense']) dict_keys(['FR_Commitment.to_do'])\n",
            "discours dict_keys(['FR_Speak_on_topic']) dict_keys(['FR_Speak_on_topic'])\n",
            "prononcer dict_keys(['FR_Expressing_decision', 'Spelling_and_pronouncing', 'FR_Making_speech']) dict_keys(['FR_Making_speech', 'Other_sense'])\n",
            "atténuation dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "laisser_entendre dict_keys(['FR_Encoding', 'Evidence']) dict_keys(['FR_Encoding'])\n",
            "annonce dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "dialogue dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "mot_d'ordre dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "s'opposer dict_keys(['Other_sense', 'FR_Taking_sides', 'Be_in_agreement_on_assessment']) dict_keys(['FR_Taking_sides', 'Other_sense'])\n",
            "indicateur dict_keys(['Other_sense', 'Evidence']) dict_keys(['Other_sense'])\n",
            "faire_part dict_keys(['FR_Telling', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "précision dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "éprouver dict_keys(['Other_sense', 'FR_Contingency-Objective_influence']) dict_keys(['FR_Contingency-Objective_influence'])\n",
            "chanter dict_keys(['FR_Statement-manner-noise', 'Other_sense']) dict_keys(['FR_Statement-manner-noise'])\n",
            "conception dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "accentuer dict_keys(['FR_Contingency-Objective_influence', 'Other_sense']) dict_keys(['FR_Contingency-Objective_influence'])\n",
            "conséquence dict_keys(['Causation']) dict_keys(['Causation'])\n",
            "supposer dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense'])\n",
            "paie dict_keys(['Commerce_pay']) dict_keys(['Commerce_pay'])\n",
            "déterminer dict_keys(['Other_sense', 'Causation', 'FR_Purpose']) dict_keys(['Other_sense', 'FR_Purpose'])\n",
            "aviser dict_keys(['Other_sense']) dict_keys(['FR_Telling'])\n",
            "explication dict_keys(['FR_Speak_on_topic', 'Explaining_the_facts', 'Quarreling']) dict_keys(['FR_Speak_on_topic'])\n",
            "joindre dict_keys(['Other_sense']) dict_keys(['Other_sense', 'FR_Contacting'])\n",
            "déduction dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "impression dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "sinon dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "expression dict_keys(['Other_sense', 'FR_Encoding', 'FR_Speak_on_topic']) dict_keys(['Other_sense'])\n",
            "communiquer dict_keys(['FR_Telling', 'FR_Chatting-Discussion', 'Other_sense']) dict_keys(['FR_Telling'])\n",
            "imaginer dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "exprimer dict_keys(['FR_Encoding', 'Other_sense']) dict_keys(['FR_Encoding', 'Other_sense'])\n",
            "dans_la_mesure_où dict_keys(['Causation', 'Evidence']) dict_keys(['Other_sense'])\n",
            "pousser dict_keys(['FR_Influence_of_event_on_cognizer-Subjective_influence', 'Other_sense', 'FR_Attempt_suasion.to_do']) dict_keys(['FR_Influence_of_event_on_cognizer-Subjective_influence'])\n",
            "de_façon_à dict_keys(['FR_Means_for_purpose']) dict_keys(['FR_Means_for_purpose'])\n",
            "conscient dict_keys(['FR_Awareness-Certainty-Opinion']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "s'entendre dict_keys(['Make_agreement_on_action', 'Be_in_agreement_on_action', 'Be_in_agreement_on_assessment', 'FR_Make_agreement_on_assessment', 'Other_sense']) dict_keys(['FR_Make_agreement_on_assessment'])\n",
            "régler dict_keys(['Other_sense', 'Commerce_pay', 'Commerce_buy']) dict_keys(['Other_sense'])\n",
            "gémir dict_keys(['Complaining']) dict_keys(['Complaining'])\n",
            "faire_ses_preuves dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "force_de_l'ordre dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "alimenter dict_keys(['FR_Contingency-Objective_influence', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "issue dict_keys(['Causation', 'Other_sense']) dict_keys(['Other_sense'])\n",
            "stimuler dict_keys(['FR_Contingency-Objective_influence']) dict_keys(['FR_Contingency-Objective_influence'])\n",
            "voir_le_jour dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "constat dict_keys(['FR_Statement-manner-noise', 'FR_Becoming_aware']) dict_keys(['FR_Statement-manner-noise', 'FR_Becoming_aware'])\n",
            "incertain dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "censé dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "dialogue_de_sourd dict_keys(['FR_Chatting-Discussion']) dict_keys(['FR_Chatting-Discussion'])\n",
            "pensée dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Other_sense'])\n",
            "ambition dict_keys(['FR_Purpose', 'Other_sense']) dict_keys(['FR_Purpose'])\n",
            "partisan dict_keys(['FR_Being_in_favor_of']) dict_keys(['FR_Being_in_favor_of'])\n",
            "du_fait_de dict_keys(['Evidence', 'Causation']) dict_keys(['Causation'])\n",
            "convergence dict_keys(['Other_sense', 'Be_in_agreement_on_assessment']) dict_keys(['Other_sense'])\n",
            "vision dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['Other_sense', 'FR_Awareness-Certainty-Opinion'])\n",
            "réflexe dict_keys(['Other_sense']) dict_keys(['Other_sense'])\n",
            "frais dict_keys(['FR_Spending', 'Other_sense']) dict_keys(['FR_Spending'])\n",
            "de_fait dict_keys(['Other_sense', 'Evidence', 'Causation']) dict_keys(['Other_sense'])\n",
            "nationaliser dict_keys(['Commerce_buy']) dict_keys(['Commerce_buy'])\n",
            "trancher dict_keys(['Other_sense', 'FR_Expressing_side']) dict_keys(['Other_sense'])\n",
            "perception dict_keys(['FR_Awareness-Certainty-Opinion', 'FR_Getting_money']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "désaccord dict_keys(['Be_in_agreement_on_assessment', 'FR_Being_in_favor_of']) dict_keys(['Be_in_agreement_on_assessment', 'FR_Being_in_favor_of'])\n",
            "confirmation dict_keys(['FR_Expressing_truth_of_proposition', 'Evidence', 'Ratification']) dict_keys(['Other_sense'])\n",
            "louer dict_keys(['Renting_out', 'Renting']) dict_keys(['FR_Judgment_communication'])\n",
            "reproche dict_keys(['Judgment_direct_address']) dict_keys(['Judgment_direct_address'])\n",
            "inconnu dict_keys(['FR_Awareness-Certainty-Opinion', 'Other_sense']) dict_keys(['FR_Awareness-Certainty-Opinion'])\n",
            "jugement dict_keys(['Other_sense', 'Regard', 'FR_Awareness-Certainty-Opinion']) dict_keys(['Regard'])\n",
            "nationalisation dict_keys(['Commerce_buy']) dict_keys(['Commerce_buy'])\n",
            "conversion dict_keys(['Other_sense']) dict_keys(['FR_Being_in_favor_of'])\n",
            "au_vu_de dict_keys(['FR_Reason', 'Evidence']) dict_keys(['Evidence'])\n",
            "anticipation dict_keys(['Other_sense', 'FR_Expectation']) dict_keys(['FR_Expectation'])\n",
            "formuler dict_keys(['FR_Encoding']) dict_keys(['FR_Encoding'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JbxiSKCp0X0z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "24f22df8-8f40-487e-f707-f415b547de7f"
      },
      "source": [
        "'The number of unknown associations in dev is', len(set(unknown_associations))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('The number of unknown associations in dev is', 22)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"So we got about {len(set(unknown_associations))/(len(dict_lemmes_dev.keys())-len(set(lemmes_inconnus)))*100}% of sens of lemmes that is unknown associations for dev according to train\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WrRx7VlfPeZa",
        "outputId": "a1bb09f5-e083-445e-8720-e6148cc87957"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "So we got about 4.65587044534413% of sens of lemmes that is unknown associations for dev according to train\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(set(lemmes_inconnus))/len(dict_lemmes_dev.keys())*100"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uXURMIfsVFCN",
        "outputId": "688c8fdc-ae45-495c-e3b3-fe0f79c5a86a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3.8910505836575875"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZDeGCyInz7w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca19bd45-146b-447a-e4ec-928748d6018f"
      },
      "source": [
        "(dict_most_frequent_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'créer': 'Other_sense',\n",
              " 'voir_le_jour': 'Other_sense',\n",
              " 'autoriser': 'FR_Grant_permission-Permitting',\n",
              " 'connaître': 'Other_sense',\n",
              " 'inviter': 'FR_Attempt_suasion.to_do',\n",
              " 'conseil': 'Other_sense',\n",
              " 'pour': 'Other_sense',\n",
              " 'rappeler': 'FR_Statement-manner-noise',\n",
              " 'pourquoi': 'Causation',\n",
              " 'préciser': 'FR_Statement-manner-noise',\n",
              " 'parce_que': 'Causation',\n",
              " 'inciter': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'donc': 'Evidence',\n",
              " 'oublier': 'FR_Memory-Remembering_experience-Remembering_information',\n",
              " 'découvrir': 'FR_Becoming_aware',\n",
              " 'comme': 'Other_sense',\n",
              " 'ambition': 'FR_Purpose',\n",
              " 'apporter': 'Other_sense',\n",
              " 'témoignage': 'FR_Speak_on_topic',\n",
              " 'aussi': 'Other_sense',\n",
              " 'marquer': 'Other_sense',\n",
              " 'conduire': 'Other_sense',\n",
              " 'commenter': 'FR_Speak_on_topic',\n",
              " 'mener': 'Other_sense',\n",
              " 'contrôle': 'Other_sense',\n",
              " 'remerciement': 'Judgment_direct_address',\n",
              " 'création': 'Other_sense',\n",
              " 'nomination': 'Appointing',\n",
              " 'mobile': 'Other_sense',\n",
              " 'projet': 'FR_Purpose',\n",
              " 'conception': 'Other_sense',\n",
              " 'laisser': 'Other_sense',\n",
              " 'faire': 'Other_sense',\n",
              " 'afin_de': 'FR_Means_for_purpose',\n",
              " 'fruit': 'Other_sense',\n",
              " 'entendre': 'Other_sense',\n",
              " 'conter': 'FR_Statement-manner-noise',\n",
              " 'annoncer': 'FR_Statement-manner-noise',\n",
              " 'remettre': 'Other_sense',\n",
              " 'entraînement': 'Other_sense',\n",
              " 'apprendre': 'Other_sense',\n",
              " 'naître': 'Other_sense',\n",
              " 'compter': 'Other_sense',\n",
              " 'perdre': 'Other_sense',\n",
              " 'souvenir': 'FR_Memory-Remembering_experience-Remembering_information',\n",
              " 'prêt': 'Other_sense',\n",
              " 'encourager': 'FR_Attempt_suasion.to_do',\n",
              " \"jusqu'à\": 'Other_sense',\n",
              " 'chant': 'Other_sense',\n",
              " 'voir': 'Other_sense',\n",
              " 'devoir': 'Other_sense',\n",
              " 'sembler': 'FR_Awareness-Certainty-Opinion',\n",
              " 'origine': 'Other_sense',\n",
              " \"à_l'origine_de\": 'Causation',\n",
              " \"s'apercevoir\": 'FR_Becoming_aware',\n",
              " 'perdre_connaissance': 'Other_sense',\n",
              " 'noter': 'FR_Statement-manner-noise',\n",
              " 'témoin': 'Other_sense',\n",
              " 'alerter': 'FR_Telling',\n",
              " 'en_effet': 'Evidence',\n",
              " 'indiquer': 'FR_Statement-manner-noise',\n",
              " 'expliquer': 'FR_Statement-manner-noise',\n",
              " 'comprendre': 'Other_sense',\n",
              " 'réaliser': 'Other_sense',\n",
              " 'manifestation': 'Other_sense',\n",
              " 'aborder': 'FR_Speak_on_topic',\n",
              " 'compréhension': 'Other_sense',\n",
              " 'dialogue': 'FR_Chatting-Discussion',\n",
              " 'jouer': 'Other_sense',\n",
              " 'applaudissement': 'FR_Judgment_communication',\n",
              " \"s'exprimer\": 'FR_Speak_on_topic',\n",
              " 'classement': 'Other_sense',\n",
              " 'alors': 'Other_sense',\n",
              " 'connaissance': 'Other_sense',\n",
              " 'affecter': 'FR_Contingency-Objective_influence',\n",
              " 'sinon': 'Other_sense',\n",
              " 'rendre_son_tablier': 'Other_sense',\n",
              " 'soutenir': 'Other_sense',\n",
              " 'querelle': 'Quarreling',\n",
              " 'faire_partie': 'Other_sense',\n",
              " 'empêcher': 'Preventing',\n",
              " 'demander': 'FR_Request',\n",
              " 'régler': 'Other_sense',\n",
              " 'faire_son_apparition': 'Other_sense',\n",
              " 'conscience': 'FR_Awareness-Certainty-Opinion',\n",
              " 'contre': 'Other_sense',\n",
              " 'sûr': 'FR_Awareness-Certainty-Opinion',\n",
              " 'commencer': 'Other_sense',\n",
              " 'faire_sentir': 'Other_sense',\n",
              " 'interpeller': 'Other_sense',\n",
              " 'appel_téléphonique': 'FR_Contacting',\n",
              " \"représentant_de_l'ordre\": 'Other_sense',\n",
              " 'assurer': 'Other_sense',\n",
              " 'répondre': 'Response',\n",
              " 'savoir': 'FR_Awareness-Certainty-Opinion',\n",
              " 'solde': 'Other_sense',\n",
              " 'grâce_à': 'Causation',\n",
              " 'marché': 'FR_Having_commercial_agreement',\n",
              " 'vente': 'Commerce_sell',\n",
              " 'discuter': 'FR_Chatting-Discussion',\n",
              " 'frais': 'FR_Spending',\n",
              " 'décider': 'FR_Deciding',\n",
              " 'envisager': 'FR_Purpose',\n",
              " 'soutien': 'Other_sense',\n",
              " 'permettre': 'Make_possible_to_do',\n",
              " 'lancer': 'FR_Cause_to_start-Launch_process',\n",
              " 'car': 'Evidence',\n",
              " 'raison': 'FR_Reason',\n",
              " 'refuser': 'FR_Agree_or_refuse_to_act',\n",
              " 'protester': 'Complaining',\n",
              " 'verser': 'FR_Giving_money',\n",
              " 'certain': 'Other_sense',\n",
              " 'condition': 'Other_sense',\n",
              " 'attirer': 'Other_sense',\n",
              " 'trouver': 'Other_sense',\n",
              " 'reconnaître': 'FR_Statement-manner-noise',\n",
              " 'subir': 'FR_Support_verb',\n",
              " 'écrire': 'FR_Statement-manner-noise',\n",
              " 'dire': 'FR_Statement-manner-noise',\n",
              " 'reprise': 'Other_sense',\n",
              " 'découler': 'Causation',\n",
              " 'puisque': 'Evidence',\n",
              " 'montrer': 'Evidence',\n",
              " 'responsable': 'Other_sense',\n",
              " 'conseiller': 'Other_sense',\n",
              " 'afin_que': 'FR_Means_for_purpose',\n",
              " 'point_de_vue': 'Other_sense',\n",
              " 'critique': 'Other_sense',\n",
              " 'pousser': 'Other_sense',\n",
              " 'contacter': 'FR_Contacting',\n",
              " 'ouvrir': 'Other_sense',\n",
              " 'confier': 'Other_sense',\n",
              " 'réaction': 'Response',\n",
              " 'conséquence': 'Causation',\n",
              " 'réponse': 'Communication_response',\n",
              " 'plaider': 'FR_Attempt_suasion.legitimacy',\n",
              " 'reprocher': 'Judgment_direct_address',\n",
              " 'dénoncer': 'FR_Judgment_communication',\n",
              " 'entamer': 'FR_Cause_to_start-Launch_process',\n",
              " 'signaler': 'FR_Statement-manner-noise',\n",
              " 'déclarer': 'FR_Statement-manner-noise',\n",
              " 'prévoir': 'Other_sense',\n",
              " 'hommage': 'FR_Judgment_communication',\n",
              " 'effet': 'Causation',\n",
              " 'menacer': 'Other_sense',\n",
              " 'disposer': 'Other_sense',\n",
              " 'fonction': 'Other_sense',\n",
              " 'alerte': 'FR_Telling',\n",
              " 'appel': 'Other_sense',\n",
              " 'prévenir': 'FR_Telling',\n",
              " 'suite': 'Other_sense',\n",
              " 'reprendre': 'Other_sense',\n",
              " 'preuve_que': 'Evidence',\n",
              " 'cotisation': 'Commerce_pay',\n",
              " 'idée': 'Other_sense',\n",
              " 'appeler': 'Referring_by_name',\n",
              " 'raconter': 'FR_Statement-manner-noise',\n",
              " 'faire_part': 'FR_Telling',\n",
              " 'conclusion': 'Coming_to_believe',\n",
              " 'favorable': 'Other_sense',\n",
              " 'prendre_connaissance': 'Other_sense',\n",
              " 'pour_que': 'Other_sense',\n",
              " 'décision': 'FR_Deciding',\n",
              " 'sachant_que': 'Other_sense',\n",
              " \"faire_l'objet\": 'Other_sense',\n",
              " 'se_prononcer': 'FR_Expressing_side',\n",
              " 'à_raison_de': 'Other_sense',\n",
              " 'porter': 'Other_sense',\n",
              " 'accord': 'Be_in_agreement_on_action',\n",
              " 'tenir': 'Other_sense',\n",
              " 'faire_confiance': 'Other_sense',\n",
              " 'atteindre': 'Other_sense',\n",
              " 'remercier': 'Judgment_direct_address',\n",
              " 'constat': 'FR_Becoming_aware',\n",
              " 'forcer': 'FR_Reason',\n",
              " 'motif': 'FR_Reason',\n",
              " 'contraindre': 'FR_Reason',\n",
              " 'entraîner': 'Causation',\n",
              " 'questionner': 'Questioning',\n",
              " 'se_souvenir': 'FR_Memory-Remembering_experience-Remembering_information',\n",
              " 'apprécier': 'Judgment',\n",
              " 'faire_remarquer': 'FR_Telling',\n",
              " 'rapport': 'FR_Statement-manner-noise',\n",
              " 'résumé': 'Summarizing',\n",
              " 'intention': 'FR_Purpose',\n",
              " 'discussion': 'FR_Chatting-Discussion',\n",
              " 'dépendre': 'FR_Contingency-Objective_influence',\n",
              " 'produire': 'Other_sense',\n",
              " 'impliquer': 'Other_sense',\n",
              " 'observer': 'FR_Becoming_aware',\n",
              " 'approuver': 'Ratification',\n",
              " 'estimer': 'FR_Awareness-Certainty-Opinion',\n",
              " 'conclure': 'FR_Support_verb',\n",
              " 'recommander': 'FR_Attempt_suasion.to_do',\n",
              " 'autorisation': 'FR_Grant_permission-Permitting',\n",
              " 'indication': 'Other_sense',\n",
              " 'se_justifier': 'Other_sense',\n",
              " 'par_conséquent': 'Causation',\n",
              " 'en_raison_de': 'Causation',\n",
              " 'produit': 'Other_sense',\n",
              " 'contrôler': 'Other_sense',\n",
              " 'dès_lors': 'Evidence',\n",
              " 'convenir': 'Other_sense',\n",
              " 'symptôme': 'Evidence',\n",
              " 'soupçonner': 'FR_Awareness-Certainty-Opinion',\n",
              " 'rapporter': 'FR_Statement-manner-noise',\n",
              " 'faire_face': 'Other_sense',\n",
              " 'informer': 'FR_Telling',\n",
              " 'signe': 'Other_sense',\n",
              " 'résultat': 'Other_sense',\n",
              " 'penser': 'FR_Awareness-Certainty-Opinion',\n",
              " 'du_fait_de': 'Causation',\n",
              " \"s'attendre\": 'FR_Expectation',\n",
              " 'selon': 'FR_Attributed_information',\n",
              " 'constater': 'FR_Becoming_aware',\n",
              " 'perte': 'Earnings_and_losses',\n",
              " 'mentionner': 'FR_Statement-manner-noise',\n",
              " 'résumer': 'Summarizing',\n",
              " 'inconnu': 'FR_Awareness-Certainty-Opinion',\n",
              " 'détecter': 'FR_Becoming_aware',\n",
              " 'dans_la_mesure_où': 'Evidence',\n",
              " 'ainsi': 'Other_sense',\n",
              " 'induire': 'Causation',\n",
              " 'prouver': 'FR_Proving',\n",
              " 'recommandation': 'FR_Attempt_suasion.to_do',\n",
              " 'exiger': 'FR_Request',\n",
              " 'source': 'Other_sense',\n",
              " 'distribution': 'Carry_goods',\n",
              " 'résulter': 'Causation',\n",
              " 'répéter': 'Other_sense',\n",
              " 'révéler': 'Evidence',\n",
              " 'démontrer': 'FR_Proving',\n",
              " 'relever': 'Other_sense',\n",
              " 'commercialiser': 'Carry_goods',\n",
              " 'ajouter': 'FR_Statement-manner-noise',\n",
              " \"jusqu'à_ce_que\": 'Other_sense',\n",
              " 'prélever': 'FR_Getting_money',\n",
              " 'étiquetage': 'Other_sense',\n",
              " 'vue': 'Other_sense',\n",
              " 'justification': 'FR_Justifying',\n",
              " 'mention': 'FR_Statement-manner-noise',\n",
              " 'doute': 'Other_sense',\n",
              " 'remarquer': 'FR_Becoming_aware',\n",
              " 'déterminer': 'Other_sense',\n",
              " 'dû': 'Causation',\n",
              " 'incidence': 'Other_sense',\n",
              " 'aboutir': 'Causation',\n",
              " 'requérir': 'Other_sense',\n",
              " 'provoquer': 'Causation',\n",
              " 'symptomatique': 'Other_sense',\n",
              " 'facteur': 'Causation',\n",
              " 'éviter': 'Preventing',\n",
              " 'classer': 'Categorization',\n",
              " 'commercialisation': 'Carry_goods',\n",
              " 'incertain': 'Other_sense',\n",
              " 'prévention': 'Preventing',\n",
              " 'confirmer': 'FR_Expressing_truth_of_proposition',\n",
              " 'cause': 'Other_sense',\n",
              " 'refléter': 'Evidence',\n",
              " 'convertir': 'Other_sense',\n",
              " 'dites': 'FR_Statement-manner-noise',\n",
              " 'menace': 'Other_sense',\n",
              " 'fonder': 'Other_sense',\n",
              " 'exposé': 'FR_Speak_on_topic',\n",
              " 'à_des_fins_de': 'FR_Means_for_purpose',\n",
              " 'objectif': 'FR_Purpose',\n",
              " 'interdire': 'FR_Grant_permission-Permitting',\n",
              " 'rembourser': 'Repayment',\n",
              " 'douter': 'FR_Awareness-Certainty-Opinion',\n",
              " 'considérer': 'FR_Awareness-Certainty-Opinion',\n",
              " 'allusion': 'FR_Speak_on_topic',\n",
              " 'débat': 'FR_Chatting-Discussion',\n",
              " 'fondement': 'Other_sense',\n",
              " 'résolution': 'Other_sense',\n",
              " \"voir_d'un_bon_oeil\": 'FR_Being_in_favor_of',\n",
              " 'rejeter': 'FR_Taking_sides',\n",
              " 'condamnation': 'Other_sense',\n",
              " 'interpréter': 'FR_Awareness-Certainty-Opinion',\n",
              " 'opinion': 'Other_sense',\n",
              " 'opposition': 'Other_sense',\n",
              " 'partisan': 'FR_Being_in_favor_of',\n",
              " 'croire': 'FR_Awareness-Certainty-Opinion',\n",
              " 'faire_preuve': 'Other_sense',\n",
              " 'opposé': 'Other_sense',\n",
              " 'se_rappeler': 'FR_Memory-Remembering_experience-Remembering_information',\n",
              " 'de_ce_fait': 'Causation',\n",
              " 'parler': 'FR_Speak_on_topic',\n",
              " 'remarque': 'FR_Statement-manner-noise',\n",
              " 'avis': 'FR_Awareness-Certainty-Opinion',\n",
              " 'regretter': 'FR_Taking_sides',\n",
              " 'souligner': 'Convey_importance',\n",
              " 'réagir': 'Response',\n",
              " 'condamner': 'Other_sense',\n",
              " 'faire_un_geste': 'Other_sense',\n",
              " \"d'accord\": 'Be_in_agreement_on_action',\n",
              " 'obliger': 'FR_Reason',\n",
              " \"c'est_pourquoi\": 'Causation',\n",
              " 'diviser': 'Other_sense',\n",
              " 'promettre': 'FR_Commitment.to_do',\n",
              " 'admettre': 'FR_Awareness-Certainty-Opinion',\n",
              " 'débattre': 'FR_Chatting-Discussion',\n",
              " 'étant_donné_que': 'Evidence',\n",
              " 'toucher': 'FR_Contingency-Objective_influence',\n",
              " 'argument': 'FR_Attempt_supported_suasion',\n",
              " 'argumenter': 'FR_Attempt_supported_suasion',\n",
              " 'concéder': 'FR_Statement-manner-noise',\n",
              " 'accepter': 'FR_Taking_sides',\n",
              " 'affirmation': 'FR_Statement-manner-noise',\n",
              " 'demande': 'Other_sense',\n",
              " 'déclaration': 'FR_Statement-manner-noise',\n",
              " 'concevoir': 'Other_sense',\n",
              " 'règlement': 'Other_sense',\n",
              " 'en_vue_de': 'FR_Means_for_purpose',\n",
              " 'en_conséquence': 'Causation',\n",
              " 'soulever': 'FR_Encoding',\n",
              " 'divergence': 'Be_in_agreement_on_assessment',\n",
              " 'justifier': 'FR_Justifying',\n",
              " \"d'avis\": 'FR_Awareness-Certainty-Opinion',\n",
              " 'faire_défaut': 'Other_sense',\n",
              " 'preuve': 'Evidence',\n",
              " 'accuser': 'FR_Judgment_communication',\n",
              " 'faute_de': 'Causation',\n",
              " 'propos': 'FR_Speak_on_topic',\n",
              " 'discours': 'FR_Speak_on_topic',\n",
              " 'en_vertu_de': 'FR_Reason',\n",
              " 'à_proprement_parler': 'Other_sense',\n",
              " 'contradiction': 'Other_sense',\n",
              " 'vu_que': 'Evidence',\n",
              " 'démentir': 'FR_Expressing_truth_of_proposition',\n",
              " 'défense': 'Other_sense',\n",
              " 'ordre_du_jour': 'Other_sense',\n",
              " 'faire_état': 'FR_Statement-manner-noise',\n",
              " 'exprimer': 'FR_Encoding',\n",
              " 'merci': 'Judgment_direct_address',\n",
              " 'exposer': 'Other_sense',\n",
              " 'refus': 'FR_Agree_or_refuse_to_act',\n",
              " 'défendre': 'Other_sense',\n",
              " 'payer': 'Commerce_pay',\n",
              " 'complimenter': 'Judgment_direct_address',\n",
              " 'à_son_avis': 'FR_Awareness-Certainty-Opinion',\n",
              " 'citer': 'Adducing',\n",
              " 'débourser': 'FR_Spending',\n",
              " 'exigence': 'FR_Request',\n",
              " 'coûter': 'FR_Spending',\n",
              " 'engagement': 'FR_Commitment.to_do',\n",
              " 'commerce': 'FR_Commerce_scenario',\n",
              " 'convaincre': 'FR_Cognizer_affecting.veracity',\n",
              " 'acquisition': 'Commerce_buy',\n",
              " 'assurance': 'Other_sense',\n",
              " 'féliciter': 'Judgment_direct_address',\n",
              " 'effectivement': 'Other_sense',\n",
              " \"s'adresser\": 'FR_Speak_on_topic',\n",
              " 'faire_savoir': 'FR_Telling',\n",
              " 'résoudre': 'Other_sense',\n",
              " 'promulguer': 'Ratification',\n",
              " 'interdiction': 'FR_Grant_permission-Permitting',\n",
              " 'se_faire_attendre': 'Other_sense',\n",
              " 'attente': 'Other_sense',\n",
              " 'divulguer': 'Reveal_secret',\n",
              " 'importer': 'Importing',\n",
              " 'lancement': 'Other_sense',\n",
              " 'coût': 'FR_Spending',\n",
              " 'supposer': 'Other_sense',\n",
              " 'initier': 'FR_Cause_to_start-Launch_process',\n",
              " 'évoquer': 'FR_Speak_on_topic',\n",
              " 'à_cause_de': 'Causation',\n",
              " 'rendre': 'Causation',\n",
              " 'paiement': 'Commerce_pay',\n",
              " 'imputer': 'FR_Attributing_cause',\n",
              " 'se_flatter': 'Bragging',\n",
              " 'sceptique': 'FR_Awareness-Certainty-Opinion',\n",
              " 'requête': 'FR_Request',\n",
              " 'commentaire': 'FR_Speak_on_topic',\n",
              " 'témoigner': 'Evidence',\n",
              " 'permission': 'FR_Grant_permission-Permitting',\n",
              " 'préconiser': 'FR_Attempt_suasion.to_do',\n",
              " 'expression': 'Other_sense',\n",
              " 'nier': 'FR_Expressing_truth_of_proposition',\n",
              " 'exportation': 'Exporting',\n",
              " 'ordre': 'Other_sense',\n",
              " 'paraître': 'FR_Awareness-Certainty-Opinion',\n",
              " 'polémique': 'Quarreling',\n",
              " 'jugement': 'Other_sense',\n",
              " 'acquitter': 'Commerce_pay',\n",
              " 'protestation': 'Complaining',\n",
              " 'juger': 'FR_Awareness-Certainty-Opinion',\n",
              " \"s'inspirer\": 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " \"s'appeler\": 'FR_Being_named',\n",
              " 'communication': 'Other_sense',\n",
              " \"d'après\": 'FR_Attributed_information',\n",
              " 'accusation': 'FR_Judgment_communication',\n",
              " 'mépris': 'Judgment',\n",
              " 'infirmer': 'Evidence',\n",
              " 'vendre': 'Commerce_sell',\n",
              " 'dévoiler': 'Reveal_secret',\n",
              " 'dans_le_but_de': 'FR_Means_for_purpose',\n",
              " 'discréditer': 'FR_Judgment_communication',\n",
              " 'contester': 'FR_Taking_sides',\n",
              " 'affirmer': 'FR_Statement-manner-noise',\n",
              " 'vu': 'Evidence',\n",
              " 'ordonner': 'FR_Request',\n",
              " 'sur_la_demande_de': 'Response',\n",
              " 'rémunérer': 'Commerce_buy',\n",
              " 'présomption': 'FR_Awareness-Certainty-Opinion',\n",
              " 'prononcer': 'FR_Expressing_decision',\n",
              " 'valider': 'Ratification',\n",
              " 'suspecter': 'FR_Awareness-Certainty-Opinion',\n",
              " 'soupçon': 'FR_Awareness-Certainty-Opinion',\n",
              " 'au_motif_que': 'FR_Reason',\n",
              " 'notifier': 'FR_Telling',\n",
              " 'révélation': 'Reveal_secret',\n",
              " 'réclamer': 'FR_Request',\n",
              " 'marque': 'Other_sense',\n",
              " 'mépriser': 'Judgment',\n",
              " 'faire_appel': 'Other_sense',\n",
              " 'engager': 'Other_sense',\n",
              " 'désapprouver': 'Judgment',\n",
              " 'rallier': 'FR_Being_in_favor_of',\n",
              " 'nommer': 'Appointing',\n",
              " 'déclenchement': 'FR_Cause_to_start-Launch_process',\n",
              " 'persuader': 'FR_Awareness-Certainty-Opinion',\n",
              " 'douteux': 'Other_sense',\n",
              " 'transaction': 'FR_Commercial_transaction',\n",
              " 'objection': 'Communication_response',\n",
              " 'contrat': 'FR_Having_commercial_agreement',\n",
              " 'acheter': 'Commerce_buy',\n",
              " 'distribuer': 'Other_sense',\n",
              " 'rémunération': 'Commerce_pay',\n",
              " 'consentir': 'FR_Agree_or_refuse_to_act_in_favor_of',\n",
              " 'motiver': 'FR_Reason',\n",
              " \"s'élever\": 'Other_sense',\n",
              " 'opposer': 'Other_sense',\n",
              " 'susciter': 'FR_Reason',\n",
              " 'certitude': 'FR_Awareness-Certainty-Opinion',\n",
              " 'ignorance': 'FR_Awareness-Certainty-Opinion',\n",
              " 'censé': 'Other_sense',\n",
              " 'désaccord': 'Be_in_agreement_on_assessment',\n",
              " 'entrevoir': 'FR_Becoming_aware',\n",
              " 'oubli': 'FR_Memory-Remembering_experience-Remembering_information',\n",
              " 'remboursement': 'Repayment',\n",
              " 'découverte': 'FR_Becoming_aware',\n",
              " 'versement': 'FR_Giving_money',\n",
              " 'déplorer': 'FR_Taking_sides',\n",
              " 'interroger': 'Questioning',\n",
              " 'inattendu': 'FR_Expectation',\n",
              " 'au_courant': 'FR_Awareness-Certainty-Opinion',\n",
              " 'influence': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'malédiction': 'FR_Judgment_communication_no_reason',\n",
              " 'rachat': 'Commerce_buy',\n",
              " 'racheter': 'Commerce_buy',\n",
              " 'citation': 'FR_Quoting',\n",
              " 'solliciter': 'FR_Request',\n",
              " 'à_la_demande_de': 'Response',\n",
              " 'en_désaccord': 'FR_Being_in_favor_of',\n",
              " 'débuter': 'Other_sense',\n",
              " 'facturation': 'Commerce_collect',\n",
              " 'foi': 'Other_sense',\n",
              " 'reverser': 'FR_Giving_money',\n",
              " 'percevoir': 'FR_Getting_money',\n",
              " 'conversation': 'FR_Chatting-Discussion',\n",
              " 'incertitude': 'FR_Awareness-Certainty-Opinion',\n",
              " 'coup_de_fil': 'FR_Contacting',\n",
              " 'achat': 'Commerce_buy',\n",
              " 'paie': 'Commerce_pay',\n",
              " 'indice': 'Other_sense',\n",
              " 'attribuer': 'Other_sense',\n",
              " 'confession': 'Reveal_secret',\n",
              " \"s'opposer\": 'FR_Taking_sides',\n",
              " 'renoncer': 'FR_Renunciation',\n",
              " 'dicter': 'Other_sense',\n",
              " 'influencer': 'FR_Contingency-Objective_influence',\n",
              " 'compte-rendu': 'Summarizing',\n",
              " 'acquérir': 'Commerce_buy',\n",
              " 'aboutissement': 'Other_sense',\n",
              " 'manifester': 'Other_sense',\n",
              " 'détailler': 'FR_Speak_on_topic',\n",
              " 'imaginer': 'Other_sense',\n",
              " 'sur_ordre_de': 'Response',\n",
              " 'avouer': 'Reveal_secret',\n",
              " 'dialoguer': 'FR_Chatting-Discussion',\n",
              " 'rajouter': 'Other_sense',\n",
              " 'explication': 'FR_Speak_on_topic',\n",
              " 'décourager': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'se_rendre_compte': 'FR_Becoming_aware',\n",
              " 'importation': 'Importing',\n",
              " 'céder': 'Commerce_sell',\n",
              " 'du_coup': 'Causation',\n",
              " 'se_résoudre': 'FR_Deciding',\n",
              " 'renvoyer_la_balle': 'Other_sense',\n",
              " 'amener': 'FR_Reason',\n",
              " 'prétendre': 'Other_sense',\n",
              " 'répercussion': 'Causation',\n",
              " 'dépense': 'FR_Spending',\n",
              " 'assuré': 'Other_sense',\n",
              " \"c'est_-_à_-_dire\": 'Other_sense',\n",
              " 'admiration': 'Judgment',\n",
              " 'conscient': 'FR_Awareness-Certainty-Opinion',\n",
              " 'à_preuve': 'Evidence',\n",
              " 'se_targuer': 'Bragging',\n",
              " \"s'allier\": 'Make_agreement_on_action',\n",
              " 'décrier': 'FR_Judgment_communication',\n",
              " 'au_point_que': 'Causation',\n",
              " 'tolérer': 'FR_Being_in_favor_of',\n",
              " 'prélèvement': 'FR_Getting_money',\n",
              " 'prendre_sa_source': 'Causation',\n",
              " 'privatisation': 'Commerce_sell',\n",
              " 'conséquent': 'Other_sense',\n",
              " 'encaisser': 'FR_Getting_money',\n",
              " 'impulsion': 'Other_sense',\n",
              " 'applaudir': 'FR_Judgment_communication',\n",
              " 'conviction': 'FR_Awareness-Certainty-Opinion',\n",
              " 'porter_-_parole': 'Other_sense',\n",
              " 'baptiser': 'Name_conferral',\n",
              " 'tabler': 'FR_Reliance_on_expectation',\n",
              " 'stimuler': 'FR_Contingency-Objective_influence',\n",
              " 'reproche': 'Judgment_direct_address',\n",
              " 'renchérir': 'Other_sense',\n",
              " 'renoncement': 'FR_Renunciation',\n",
              " 'critiquer': 'FR_Judgment_communication',\n",
              " 'but': 'FR_Purpose',\n",
              " 'ignorer': 'Other_sense',\n",
              " 'vanter': 'FR_Judgment_communication',\n",
              " 'gagner': 'Other_sense',\n",
              " 'inspirer': 'Other_sense',\n",
              " 'trancher': 'FR_Expressing_side',\n",
              " 'de_fait': 'Evidence',\n",
              " 'reconnaissance': 'FR_Statement-manner-noise',\n",
              " 'se_défendre': 'FR_Expressing_truth_of_proposition',\n",
              " 'atténuer': 'FR_Contingency-Objective_influence',\n",
              " 'amorcer': 'Other_sense',\n",
              " 'moteur': 'Other_sense',\n",
              " 'classification': 'Categorization',\n",
              " 'motivation': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'être_ce_à_dire': 'Other_sense',\n",
              " 'presser': 'Other_sense',\n",
              " 'engendrer': 'FR_Cause_to_start-Launch_process',\n",
              " 'avancer': 'FR_Statement-manner-noise',\n",
              " 'alliance': 'Be_in_agreement_on_action',\n",
              " 'déterminant': 'FR_Contingency-Objective_influence',\n",
              " 'escompter': 'FR_Reliance_on_expectation',\n",
              " 'au_vu_de': 'Evidence',\n",
              " 'avoir_pour_nom': 'FR_Being_named',\n",
              " 'indicateur': 'Other_sense',\n",
              " 'générer': 'Other_sense',\n",
              " 'conversion': 'Other_sense',\n",
              " \"tomber_d'accord\": 'FR_Make_agreement_on_assessment',\n",
              " 'confirmation': 'Ratification',\n",
              " 'à_la_suite_de': 'Causation',\n",
              " \"s'engager\": 'FR_Commitment.to_do',\n",
              " 'brader': 'Commerce_sell',\n",
              " 'souscrire': 'Other_sense',\n",
              " 'formulation': 'FR_Encoding',\n",
              " \"voir_d'un_mauvais_oeil\": 'FR_Being_in_favor_of',\n",
              " 'contact': 'FR_Contacting',\n",
              " 'pourparler': 'FR_Chatting-Discussion',\n",
              " 'pour_preuve': 'Evidence',\n",
              " 'de_façon_à': 'FR_Means_for_purpose',\n",
              " 'laisser_entendre': 'FR_Encoding',\n",
              " 'contrer': 'Response',\n",
              " 'congratuler': 'Judgment_direct_address',\n",
              " 'fustiger': 'FR_Judgment_communication',\n",
              " 'cotiser': 'Commerce_pay',\n",
              " 'conditionner': 'Other_sense',\n",
              " 'déterminé': 'Other_sense',\n",
              " 'prévision': 'FR_Expectation',\n",
              " 'faire_ses_preuves': 'Other_sense',\n",
              " 'donner_raison': 'Other_sense',\n",
              " 'faire_suite': 'Other_sense',\n",
              " 'réticent': 'Willingness',\n",
              " 'se_rallier': 'FR_Being_in_favor_of',\n",
              " 'infliger': 'Other_sense',\n",
              " 'enjoindre': 'FR_Request',\n",
              " 'écrit': 'Text_creation',\n",
              " 'repousser': 'Other_sense',\n",
              " 'accréditer': 'Evidence',\n",
              " 'impact': 'Causation',\n",
              " 'déclencher': 'FR_Cause_to_start-Launch_process',\n",
              " 'déduire': 'Other_sense',\n",
              " 'gain': 'Other_sense',\n",
              " 'pour_cause_de': 'Causation',\n",
              " 'parier': 'FR_Reliance_on_expectation',\n",
              " 'sentir': 'FR_Awareness-Certainty-Opinion',\n",
              " 'faire_observer': 'FR_Telling',\n",
              " \"mot_d'ordre\": 'Other_sense',\n",
              " \"c'est_à_-_dire\": 'Other_sense',\n",
              " 'imprévisible': 'FR_Expectation',\n",
              " 'écouler': 'Commerce_sell',\n",
              " 'contrainte': 'Other_sense',\n",
              " 'croyance': 'FR_Awareness-Certainty-Opinion',\n",
              " 'élogieux': 'FR_Judgment_communication',\n",
              " 'paye': 'Other_sense',\n",
              " 'écriture': 'Other_sense',\n",
              " 'impression': 'FR_Awareness-Certainty-Opinion',\n",
              " 'exporter': 'Exporting',\n",
              " 'sentiment': 'FR_Awareness-Certainty-Opinion',\n",
              " 'à_ce_point_que': 'Causation',\n",
              " 'anathème': 'FR_Judgment_communication',\n",
              " \"s'entendre\": 'Make_agreement_on_action',\n",
              " 'adhésion': 'FR_Being_in_favor_of',\n",
              " 'entériner': 'Ratification',\n",
              " 'prendre_position': 'Other_sense',\n",
              " 'perception': 'FR_Awareness-Certainty-Opinion',\n",
              " 'décliner': 'Other_sense',\n",
              " 'revendre': 'Commerce_sell',\n",
              " 'songer': 'Other_sense',\n",
              " 'adhérer': 'Other_sense',\n",
              " \"de_l'ordre_de\": 'Other_sense',\n",
              " 'issue': 'Causation',\n",
              " 'dès_lors_que': 'Other_sense',\n",
              " 'se_faire_prier': 'Other_sense',\n",
              " 'illusion': 'FR_Awareness-Certainty-Opinion',\n",
              " \"s'accorder\": 'Be_in_agreement_on_assessment',\n",
              " 'arrangement': 'Be_in_agreement_on_action',\n",
              " 'négoce': 'FR_Commerce_scenario',\n",
              " 'éprouver': 'Other_sense',\n",
              " 'attester': 'Evidence',\n",
              " 'nationaliser': 'Commerce_buy',\n",
              " 'anticipation': 'Other_sense',\n",
              " 'solder': 'Other_sense',\n",
              " 'avertir': 'FR_Telling',\n",
              " 'pressentir': 'Other_sense',\n",
              " 'vouloir_dire': 'Other_sense',\n",
              " 'écoulement': 'Commerce_sell',\n",
              " 'exalter': 'FR_Judgment_communication',\n",
              " 'serment': 'FR_Commitment.to_do',\n",
              " 'projeter': 'FR_Purpose',\n",
              " 'alléguer': 'Adducing',\n",
              " 'prôner': 'FR_Attempt_suasion.legitimacy',\n",
              " 'alimenter': 'Other_sense',\n",
              " 'au_fait': 'FR_Awareness-Certainty-Opinion',\n",
              " 'accentuer': 'FR_Contingency-Objective_influence',\n",
              " 'imputable': 'FR_Attributing_cause',\n",
              " 'à_défaut_de': 'Other_sense',\n",
              " 'plaidoyer': 'FR_Attempt_supported_suasion',\n",
              " 'à_vrai_dire': 'Other_sense',\n",
              " 'cession': 'Commerce_sell',\n",
              " 'réplique': 'FR_Statement-manner-noise',\n",
              " 'rétorquer': 'Communication_response',\n",
              " 'générateur': 'FR_Cause_to_start-Launch_process',\n",
              " 'dans_ce_but': 'FR_Means_for_purpose',\n",
              " 'précision': 'FR_Statement-manner-noise',\n",
              " 'animer': 'Other_sense',\n",
              " 'surcoût': 'FR_Spending',\n",
              " 'se_féliciter': 'FR_Taking_sides',\n",
              " 'mener_à_bien': 'Other_sense',\n",
              " 'louer': 'Renting_out',\n",
              " 'pour_ainsi_dire': 'Other_sense',\n",
              " 'prédire': 'Predicting',\n",
              " 'et_pour_cause': 'Causation',\n",
              " 'promesse': 'FR_Commitment.to_do',\n",
              " \"s'exclamer\": 'FR_Statement-manner-noise',\n",
              " 'dépenser': 'FR_Spending',\n",
              " 'causer': 'Causation',\n",
              " 'trouver_sa_source': 'Causation',\n",
              " 'illustrer': 'Evidence',\n",
              " 'donner_naissance': 'Other_sense',\n",
              " 'dessein': 'Other_sense',\n",
              " 'interrogation': 'Questioning',\n",
              " 'dédaigner': 'Judgment',\n",
              " 'rendre_compte': 'Other_sense',\n",
              " 'rejet': 'FR_Taking_sides',\n",
              " 'suggestion': 'FR_Attempt_suasion.to_do',\n",
              " 'de_sorte_que': 'Causation',\n",
              " 'prière': 'Other_sense',\n",
              " 'revente': 'Commerce_sell',\n",
              " 'en_coûter': 'FR_Spending',\n",
              " 'hypothèse': 'FR_Awareness-Certainty-Opinion',\n",
              " 'bradage': 'Commerce_sell',\n",
              " \"d'où\": 'Causation',\n",
              " 'gronder': 'Other_sense',\n",
              " \"s'insurger\": 'FR_Taking_sides',\n",
              " 'porter_le_à': 'Other_sense',\n",
              " 'mettre_sur_le_compte_de': 'FR_Attributing_cause',\n",
              " 'se_résumer': 'Other_sense',\n",
              " 'arguer': 'FR_Justifying',\n",
              " 'jurer': 'FR_Commitment.to_do',\n",
              " 'location': 'Renting_out',\n",
              " 'au_point_de': 'Causation',\n",
              " 'anticiper': 'FR_Expectation',\n",
              " 'tant_et_si_bien_que': 'Causation',\n",
              " 'se_convertir': 'FR_Being_in_favor_of',\n",
              " 'louange': 'FR_Judgment_communication',\n",
              " \"sous_l'effet_de\": 'Causation',\n",
              " 'allumer': 'Other_sense',\n",
              " 'avaliser': 'Ratification',\n",
              " 'communiquer': 'FR_Telling',\n",
              " 'annonce': 'FR_Statement-manner-noise',\n",
              " 'dits': 'Other_sense',\n",
              " \"raison_d'être\": 'Other_sense',\n",
              " 'bonifier': 'Commerce_buy',\n",
              " 'de_manière_à': 'FR_Means_for_purpose',\n",
              " 'à_cet_effet': 'FR_Means_for_purpose',\n",
              " 'désavouer': 'FR_Expressing_truth_of_proposition',\n",
              " 'ménager': 'Other_sense',\n",
              " 'prise_de_conscience': 'FR_Becoming_aware',\n",
              " 'incitation': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'acceptation': 'FR_Taking_sides',\n",
              " 'prendre_la_parole': 'FR_Speak_on_topic',\n",
              " 'à_sa_demande': 'Response',\n",
              " 'démarrer': 'Other_sense',\n",
              " \"donneur_d'ordre\": 'Other_sense',\n",
              " 'donneur_ordre': 'Other_sense',\n",
              " 'ce_être_-_à_-_dire': 'Other_sense',\n",
              " \"s'indigner\": 'FR_Taking_sides',\n",
              " 'déni': 'Other_sense',\n",
              " 'au_mépris_de': 'Other_sense',\n",
              " 'contredire': 'Evidence',\n",
              " 'import': 'Importing',\n",
              " 'export': 'Exporting',\n",
              " 'sermon': 'Judgment_direct_address',\n",
              " 'pronostiquer': 'Predicting',\n",
              " 'ambitionner': 'FR_Purpose',\n",
              " 'il_inspirer': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'dispute': 'Other_sense',\n",
              " 'acculer': 'FR_Reason',\n",
              " 'gémir': 'Complaining',\n",
              " 'réveiller': 'Other_sense',\n",
              " 'disputer': 'Other_sense',\n",
              " 'si_bien_que': 'Causation',\n",
              " 'observation': 'Other_sense',\n",
              " 'miser': 'FR_Reliance_on_expectation',\n",
              " 'en_bon_ordre': 'Other_sense',\n",
              " 'privatiser': 'Commerce_sell',\n",
              " 'répétition': 'Other_sense',\n",
              " 'astreindre': 'FR_Reason',\n",
              " 'unanime': 'Be_in_agreement_on_action',\n",
              " 'suggérer': 'FR_Attempt_suasion.to_do',\n",
              " 'amodiation': 'Renting',\n",
              " 'dans_un_but_de': 'FR_Means_for_purpose',\n",
              " 'repérer': 'FR_Becoming_aware',\n",
              " \"se_mettre_d'accord\": 'FR_Make_agreement_on_assessment',\n",
              " 'par_suite': 'Causation',\n",
              " 'proclamer': 'FR_Statement-manner-noise',\n",
              " 'se_décider': 'FR_Deciding',\n",
              " 'objecter': 'Communication_response',\n",
              " 'convier': 'FR_Attempt_suasion.to_do',\n",
              " 'vision': 'FR_Awareness-Certainty-Opinion',\n",
              " 'liquider': 'Commerce_sell',\n",
              " 'par_la_faute_de': 'Causation',\n",
              " 'bonification': 'Commerce_buy',\n",
              " 'approbation': 'Ratification',\n",
              " 'retombée': 'Causation',\n",
              " 'désignation': 'Appointing',\n",
              " 'avertissement': 'FR_Telling',\n",
              " 'stimulation': 'FR_Contingency-Objective_influence',\n",
              " 'étant_donné': 'Evidence',\n",
              " 'appelé': 'Referring_by_name',\n",
              " 'mettre_en_doute': 'FR_Expressing_truth_of_proposition',\n",
              " 'reversement': 'FR_Giving_money',\n",
              " 'du': 'Causation',\n",
              " 'prévisible': 'FR_Expectation',\n",
              " 'pronostic': 'Predicting',\n",
              " 'convergence': 'Be_in_agreement_on_assessment',\n",
              " 'haranguer': 'FR_Speak_on_topic',\n",
              " 'dit': 'FR_Being_named',\n",
              " 'riposte': 'Response',\n",
              " 'contre-attaque': 'Response',\n",
              " 'répliquer': 'Communication_response',\n",
              " 'téléphoner': 'FR_Contacting',\n",
              " \"s'arranger\": 'Make_agreement_on_action',\n",
              " 'faire_en_sorte': 'Causation',\n",
              " 'en_appeler': 'FR_Attempt_suasion.to_do',\n",
              " 'credo': 'FR_Awareness-Certainty-Opinion',\n",
              " 'inspiration': 'Other_sense',\n",
              " 'péage': 'Commerce_pay',\n",
              " 'atténuation': 'Other_sense',\n",
              " 'contestation': 'FR_Taking_sides',\n",
              " 'séquelle': 'Causation',\n",
              " 'se_plaindre': 'Complaining',\n",
              " 'affréter': 'Renting',\n",
              " 'sommer': 'FR_Request',\n",
              " 'se_vanter': 'Bragging',\n",
              " 'interprétation': 'FR_Awareness-Certainty-Opinion',\n",
              " 'décidé': 'FR_Deciding',\n",
              " 'exhorter': 'FR_Attempt_suasion.to_do',\n",
              " \"protocole_d'accord\": 'Be_in_agreement_on_action',\n",
              " 'lentille_de_contact': 'Other_sense',\n",
              " 'crier': 'FR_Statement-manner-noise',\n",
              " 'dans_quel_but': 'FR_Means_for_purpose',\n",
              " \"rappeler_à_l'ordre\": 'Judgment_direct_address',\n",
              " 'constatation': 'FR_Statement-manner-noise',\n",
              " 'présager': 'FR_Expectation',\n",
              " \"de_l'avis_de\": 'FR_Awareness-Certainty-Opinion',\n",
              " \"à_l'appel_de\": 'FR_Attempt_suasion.to_do',\n",
              " 'désaveu': 'FR_Expressing_truth_of_proposition',\n",
              " 'certifier': 'FR_Commitment.veracity',\n",
              " 'par_contrecoup': 'Causation',\n",
              " 'invitation': 'Other_sense',\n",
              " 'formuler': 'FR_Encoding',\n",
              " 'se_concerter': 'FR_Make_agreement_on_assessment',\n",
              " 'imprévu': 'FR_Expectation',\n",
              " 'pensée': 'Other_sense',\n",
              " 'à_des_fins': 'FR_Means_for_purpose',\n",
              " 'déceler': 'FR_Becoming_aware',\n",
              " 'commercer': 'FR_Commerce_scenario',\n",
              " 'aveu': 'Reveal_secret',\n",
              " 'à_force_de': 'Causation',\n",
              " 'encouragement': 'FR_Influence_of_event_on_cognizer-Subjective_influence',\n",
              " 'intuition': 'FR_Awareness-Certainty-Opinion',\n",
              " 'déduction': 'Other_sense',\n",
              " \"dans_l'optique_de\": 'FR_Means_for_purpose',\n",
              " 'discrédit': 'FR_Judgment_communication',\n",
              " 'étayer': 'Other_sense',\n",
              " 'convaincant': 'FR_Cognizer_affecting.veracity',\n",
              " 'préjugé': 'FR_Awareness-Certainty-Opinion',\n",
              " 'argumentation': 'FR_Attempt_supported_suasion',\n",
              " 'préjuger': 'FR_Expectation',\n",
              " 'validation': 'Other_sense',\n",
              " 'se_perdre': 'Other_sense',\n",
              " 'tractation': 'FR_Chatting-Discussion',\n",
              " 'être_fonction_de': 'FR_Contingency-Objective_influence',\n",
              " 'initiateur': 'FR_Cause_to_start-Launch_process',\n",
              " \"force_de_l'ordre\": 'Other_sense',\n",
              " 'délibérer': 'Other_sense',\n",
              " 'contrecoup': 'Causation',\n",
              " 'à_juste_raison': 'Other_sense',\n",
              " 'évocation': 'FR_Speak_on_topic',\n",
              " 'se_lamenter': 'Complaining',\n",
              " 'discourir': 'FR_Speak_on_topic',\n",
              " 'dissuader': 'FR_Cognizer_affecting.to_do',\n",
              " 'ordre_public': 'Other_sense',\n",
              " 'gager': 'FR_Expectation',\n",
              " 'récrimination': 'Complaining',\n",
              " 'nationalisation': 'Commerce_buy',\n",
              " 'avoir_à_revendre': 'Other_sense',\n",
              " 'remémorer': 'FR_Telling',\n",
              " 'infléchir': 'FR_Contingency-Objective_influence',\n",
              " 'scepticisme': 'FR_Awareness-Certainty-Opinion',\n",
              " 'professer': 'FR_Statement-manner-noise',\n",
              " 'rétribution': 'Commerce_pay',\n",
              " 'balbutier': 'Other_sense',\n",
              " \"voir_d'un_oeil_bienveillant\": 'FR_Being_in_favor_of',\n",
              " 'sensibilisation': 'FR_Attempt_suasion.legitimacy',\n",
              " 'influer': 'FR_Contingency-Objective_influence',\n",
              " 'ferrailler': 'Quarreling',\n",
              " 'dédain': 'Judgment',\n",
              " 'induction': 'Other_sense',\n",
              " 'à_tel_point_que': 'Causation',\n",
              " 'prier': 'Other_sense',\n",
              " 'sensibiliser': 'FR_Attempt_suasion.legitimacy',\n",
              " 'riposter': 'Response',\n",
              " 'prêcher': 'FR_Attempt_suasion.legitimacy',\n",
              " 'hurler': 'FR_Statement-manner-noise',\n",
              " 'recouvrer': 'FR_Getting_money',\n",
              " 'plaidoirie': 'FR_Attempt_supported_suasion',\n",
              " 'déprécier': 'FR_Judgment_communication',\n",
              " 'avoir_raison': 'Other_sense',\n",
              " 'dénier': 'FR_Grant_permission-Permitting',\n",
              " 'par_voie_de_conséquence': 'Causation',\n",
              " 'réprimander': 'Judgment_direct_address',\n",
              " 'se_disputer': 'Quarreling',\n",
              " 'deviner': 'Coming_to_believe',\n",
              " 'à_point_nommé': 'Other_sense',\n",
              " 'injonction': 'FR_Request',\n",
              " 'vouloir_pour_preuve': 'FR_Attempt_supported_suasion',\n",
              " 'légitimation': 'FR_Justifying',\n",
              " 'en_vertu_duquel': 'Other_sense',\n",
              " 'compliment': 'Judgment_direct_address',\n",
              " 'ronchonner': 'Complaining',\n",
              " 'dissuasion': 'FR_Cognizer_affecting.to_do',\n",
              " 'donner_à_penser': 'Evidence',\n",
              " 'délibération': 'FR_Chatting-Discussion',\n",
              " 'dialogue_de_sourd': 'FR_Chatting-Discussion',\n",
              " 'aviser': 'Other_sense',\n",
              " 'réflexe': 'Other_sense',\n",
              " 'ressort': 'Other_sense',\n",
              " 'ralliement': 'FR_Being_in_favor_of',\n",
              " 'cataloguer': 'Categorization',\n",
              " \"avec_l'objectif_de\": 'FR_Means_for_purpose',\n",
              " 'dénigrement': 'FR_Judgment_communication',\n",
              " 'marmonner': 'FR_Statement-manner-noise',\n",
              " 'donner_à_croire': 'FR_Attempt_suasion.veracity',\n",
              " 'réclamation': 'Other_sense',\n",
              " 'proclamation': 'FR_Statement-manner-noise',\n",
              " 'résolu': 'FR_Purpose',\n",
              " 'présumer': 'FR_Awareness-Certainty-Opinion',\n",
              " \"sur_l'injonction_de\": 'Response',\n",
              " \"raison_d'état\": 'Other_sense',\n",
              " 'se_confier': 'Reveal_secret',\n",
              " 'maudire': 'FR_Judgment_communication',\n",
              " 'chanter': 'FR_Statement-manner-noise',\n",
              " 'exaltation': 'Other_sense',\n",
              " 'déchaîner': 'Other_sense',\n",
              " 'renonciation': 'FR_Renunciation',\n",
              " 'facturer': 'Commerce_collect',\n",
              " 'brouille': 'Quarreling',\n",
              " 'sommation': 'FR_Request',\n",
              " 'causerie': 'FR_Chatting-Discussion',\n",
              " 'influençable': 'Other_sense',\n",
              " 'chanter_les_louanges': 'FR_Judgment_communication',\n",
              " 'désapprobation': 'Judgment',\n",
              " 'apostropher': 'FR_Hail',\n",
              " 'confesser': 'Other_sense',\n",
              " \"s'aviser\": 'Other_sense',\n",
              " 'joindre': 'Other_sense',\n",
              " 'reparler': 'FR_Speak_on_topic',\n",
              " 'disposé': 'Willingness',\n",
              " 'persuasion': 'FR_Cognizer_affecting.veracity',\n",
              " 'récuser': 'FR_Taking_sides',\n",
              " 'sonder': 'Questioning',\n",
              " 'faire_valoir': 'Other_sense',\n",
              " 'recouvrement': 'FR_Getting_money',\n",
              " \"c'est_que\": 'Explaining_the_facts',\n",
              " 'proférer': 'FR_Making_speech',\n",
              " 'promulgation': 'Ratification',\n",
              " 'épingler': 'FR_Judgment_communication',\n",
              " 'corroborer': 'Evidence',\n",
              " 'plus_que_de_raison': 'Other_sense',\n",
              " 'divulgation': 'Reveal_secret',\n",
              " 'en_ordre': 'Other_sense',\n",
              " 'classifier': 'Categorization',\n",
              " 'estime': 'Judgment',\n",
              " 'du_fait_que': 'Causation',\n",
              " 'faire_naître': 'Other_sense',\n",
              " 'implorer': 'FR_Request',\n",
              " 'grogner': 'FR_Statement-manner-noise'}"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XPbOlFK9nLoU"
      },
      "source": [
        "## Modèle et tokenization de type *BERT\n",
        "\n",
        "On va utiliser un modèle pré-entraîné de type *BERT, en passant par le module \"transformers\" d'huggingface."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4DfehySexyR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "539cb8c1-b0be-4b27-a9f8-5a2f09105fa7"
      },
      "source": [
        "try:\n",
        "  import transformers\n",
        "except ImportError:\n",
        "  !pip install transformers\n",
        "  \n",
        "# les modules permettant de charger un modèle (resp. un tokenizer / une config)\n",
        "# et de repérer le type d'instance automatiquement d'après le nom du modèle\n",
        "from transformers import AutoModel, AutoTokenizer, AutoConfig"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 29.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.8.2)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n",
            "\u001b[K     |████████████████████████████████| 182 kB 67.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 68.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.11.1 tokenizers-0.13.2 transformers-4.25.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JJiKhfxhexyV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408,
          "referenced_widgets": [
            "6ff2cfe533b94ec19835478e945c9b46",
            "4f9f8dec224c49f8a8ac78dc8f829e43",
            "7f29a31c548b4d96a2a9540e86ef287c",
            "60c02f192ada47cda73bd7a4be157721",
            "cd2d79bd25e54f8c972a75a7978298d3",
            "742ecfbb7baf4a6ea90267cc7147d243",
            "9f03a8a2db4e4a5b9d525059926e5807",
            "bebc6c90a5c24dd481f81637600589e8",
            "90e3bfbb8c424fe2a303443bc340729e",
            "4fccc64ad24946448f653e54eeb44f2c",
            "d1f1c466e239451292626a619a0c0e3d",
            "ac3180b9f6d244949984240a3451ab60",
            "aa23ceeab7274e01a3f5143a7da3c912",
            "12bef55836424effb836e42de88a93b7",
            "244040fdba2b4535b5db6c39d701006d",
            "497813ee61e448358edb1626836f8c02",
            "1bd4c49ca3cd4dadaa0600e881d6d593",
            "1bda3b1158084d9fb576606e7e0f34de",
            "7152dca9bf204b8e93ca44769bfc5e07",
            "a0ecbeee6ad7450cabb1f50a77dd1701",
            "b0579c7432e34b7586437d41c5eb793f",
            "88902c77a50c4362911498301946fbff",
            "e4d45bd05f6f44abb5118585df3e9ac4",
            "d63b066a7d1a4238bb7f1caa64b481f1",
            "9991b634e65b4393b8c692f82a7f5ed7",
            "2b615107feb2419abb52d22955443449",
            "0272af9a301e4869a6503af97431a65a",
            "dde8fee0f7ec413fb1e8d30b1fb0beb4",
            "82a77714a2af4232b4cba8869f0409f3",
            "5cc4c71304294efb9c1fe22b7c0cc755",
            "773fa3f733304f888d3bd0d1aa84e385",
            "011e44fc801c4ee3b7c398899f9cdabf",
            "e7d402755f6848b1b81d31a5b8ab5595",
            "73a6436d36334ad2a6c774efada1647c",
            "c13a831a8f9644cab1b21ea80b6637d8",
            "b7b7bc70dab64924a7c5765130154c1b",
            "381d24095a614ffa9e45c08fa11c0fd4",
            "b39bb0936c844685a54e7f2da3a90aa0",
            "51d5fe625fd44591af86b7c988168004",
            "cf9fe539b6fc44078f7b3159e1c9decd",
            "ce3dcf6f382f4125af6002a88f4aa277",
            "e6087e53d7e34bf5ae333fdd5cb910c4",
            "dfed4c93d7114446b3a63a967db64809",
            "112db6d4bbc34d0aaa66b1b7fa7ab52c"
          ]
        },
        "outputId": "47b7470a-4ab5-432e-d44c-4c6014770755"
      },
      "source": [
        "# On choisit de travailler avec le modèle FlauBERT\n",
        "# cf. liste des modèles dispos : https://huggingface.co/transformers/pretrained_models.html\n",
        "%pip install sacremoses\n",
        "# on charge ici le tokenizer Flaubert\n",
        "# et la config du modèle\n",
        "flaubert_tokenizer = AutoTokenizer.from_pretrained(\"flaubert/flaubert_base_cased\")\n",
        "flaubert_config = AutoConfig.from_pretrained(\"flaubert/flaubert_base_cased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[K     |████████████████████████████████| 880 kB 28.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.8/dist-packages (from sacremoses) (2022.6.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.8/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (from sacremoses) (1.2.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from sacremoses) (4.64.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=da4639428be2b8810f2cda68ca9814ad1eec2368630f72eac23aa7ddb39eb210\n",
            "  Stored in directory: /root/.cache/pip/wheels/82/ab/9b/c15899bf659ba74f623ac776e861cf2eb8608c1825ddec66a4\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses\n",
            "Successfully installed sacremoses-0.0.53\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6ff2cfe533b94ec19835478e945c9b46"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.50k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ac3180b9f6d244949984240a3451ab60"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.56M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e4d45bd05f6f44abb5118585df3e9ac4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/896k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "73a6436d36334ad2a6c774efada1647c"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rMmtutNBexyQ"
      },
      "source": [
        "### Encodage des données (correspondance entre rang de mot et rang de token \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "Pour pouvoir utiliser un modèle *BERT pré-entraîné, il faut\n",
        "- utiliser la même tokenisation en tokens (potentiellement des sous-mots) que celle utilisée à l'entraînement du modèle\n",
        "- convertir les séquences de tokens en séquences d'ids de tokens \n",
        "- et maintenir un lien entre les rangs de mot dans la phrase (dont le rang du target) et les rangs de tokens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GuLEbwS_tVtZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01ca691b-1f63-4e1d-9b09-b3ff13d01884"
      },
      "source": [
        "from nltk.tokenize import WordPunctTokenizer\n",
        "class WSDEncoder:\n",
        "    def __init__(self, tokenizer, config):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.config = config # pour récupérer les indices des tokens spéciaux\n",
        "    \n",
        "    def new_ranks(self, sentences, tg_works):\n",
        "\n",
        "      tg_trks = []#[4,3,..]\n",
        "      phrases = []#[['Con','séqu', 'emment</w>','leurs</w>','codes</w>','compr','endraient</w>','des</w>','erreurs</w>','.</w>'], [...]... ]\n",
        "      \n",
        "\n",
        "      for sent, rank in zip(sentences, tg_works):\n",
        "        has_seen = False\n",
        "       #\"Conséquemment leurs codes comprendraient des erreurs .\",\n",
        "        word_gold = sent[rank]\n",
        "        phrase = []\n",
        "        for index, word in enumerate(sent):\n",
        "          if word == word_gold and has_seen == False:\n",
        "            tg_trks.append(len(phrase)+1)#+1 Because of the eventual 'beginning of sentence' token\n",
        "            has_seen = True\n",
        "          phrase.extend(flaubert_tokenizer.tokenize(word))\n",
        "        phrases.append(phrase)\n",
        "      \n",
        "      return phrases, tg_trks\n",
        " \n",
        "    \n",
        "    def encode(self, sentences, tg_wrks, max_length=350, verbose=False, is_split_into_words=True):\n",
        "      \"\"\" \n",
        "      Input: \n",
        "        - sentences : list of sentences\n",
        "           -- if is_split_into_words:\n",
        "              sentences are already split into words \n",
        "              (hence sentences = list of word strings [[w1, w2, w3], [w1, w2]...])\n",
        "           -- otherwise, sentences are to split on spaces to get words\n",
        "\n",
        "        - tg_wrks : list of the ranks of target words\n",
        "          (one rank per sentence, starting at 0 in a sentence)\n",
        "        - max_length : maximum length in number of tokens\n",
        "\n",
        "      Returns:\n",
        "        - tid_seqs : the sentences padded/truncated so that each contains max_length token ids\n",
        "        - first_trk_of_targets : for each sentence, \n",
        "                                 the rank in corresponding tid_seq\n",
        "                                 of the first token of the target word\n",
        "      Example\n",
        "      sentences = ['Conséquemment , nous comprendrions .']\n",
        "      tg_wrks = [3]\n",
        "\n",
        "      if the sentence is tokenized into \n",
        "        '<s>', 'Con', 'séqu', 'emment</w>', ',</w>', 'nous</w>', 'compr', 'end', 'rions</w>', '.</w>' ....\n",
        "      the first token rank of the target \"comprendrions\" is 6 ('compr')\n",
        "\n",
        "      \"\"\"\n",
        "      if is_split_into_words == False:\n",
        "        sentences = [sentence.split() for sentence in sentences] #Splitting sentences on spaces\n",
        "      \n",
        "     \n",
        "\n",
        "      phrases, first_trk_of_targets = self.new_ranks(sentences, tg_wrks)\n",
        "      \n",
        "      \n",
        "      tokenized = []\n",
        "   \n",
        "      for phrase in phrases:\n",
        "         tokenized.append(flaubert_tokenizer.encode(phrase,add_special_tokens = True,truncation = True, max_length = max_length, padding = 'max_length', pad_to_max_length = True))\n",
        "     \n",
        "      #Encoding lemmas\n",
        "\n",
        "      \n",
        "      # TODO HERE : encoding method\n",
        "\n",
        "      # Indications:\n",
        "      # 1. apply flaubert tokenization word per word, and build\n",
        "      #    tid_seqs first without padding / truncation nor special tokens,\n",
        "      #    and keep track of token rank of first token of target word\n",
        "      # 2. then truncate and pad, and add special symbols\n",
        "      # (write several methods for easier reading)\n",
        "      \n",
        "      return tokenized, first_trk_of_targets\n",
        "\n",
        "encoder = WSDEncoder(flaubert_tokenizer, flaubert_config)\n",
        "\n",
        "# test encoder\n",
        "test_sents = [\"Conséquemment leurs codes comprendraient des erreurs .\",\n",
        "            \"J' essaie de comprendre les transformers .\",  \n",
        "            \"Il n' a pas bien compris le code !\"]\n",
        "# les mots target sont les occurrences de du verbe \"comprendre\"\n",
        "test_tg_wrks = [3, 3, 5] # En réalité pour la première phrase comprendraient se situe au troisième\n",
        "\n",
        "# TODO: décommenter pour tester votre méthode encode\n",
        "\n",
        "# 1. Not add padding and not add special tokens\n",
        "\n",
        "print(\"Not add padding and not add special tokens : \")\n",
        "tid_seqs, first_trk_of_targets = encoder.encode(test_sents, test_tg_wrks, max_length=100,is_split_into_words=False)\n",
        "#print(len(tid_seqs),\" \",len(first_trk_of_targets))\n",
        "for tid_seq, ft in zip(tid_seqs, first_trk_of_targets):\n",
        "    print(\"Len = %d target token rank = %d tid_seq = %s\" % (len(tid_seq), ft, str(tid_seq))) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Not add padding and not add special tokens : \n",
            "Len = 100 target token rank = 6 tid_seq = [0, 1198, 17358, 13299, 121, 5677, 18719, 16724, 23, 3842, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n",
            "Len = 100 target token rank = 4 tid_seq = [0, 158, 5213, 15, 965, 22, 14659, 896, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n",
            "Len = 100 target token rank = 6 tid_seq = [0, 59, 51, 34, 42, 83, 681, 20, 1138, 82, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "phrase=flaubert_tokenizer.tokenize(\"Conséquemment leurs codes comprendraient des erreurs .\")\n",
        "flaubert_tokenizer.encode(phrase,add_special_tokens = True,truncation = True, max_length = 15, padding = 'max_length', pad_to_max_length = True),phrase\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mksG5Z9Ue0mL",
        "outputId": "398f6465-401b-4603-bc81-28adb2de94b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([0, 1198, 17358, 13299, 121, 5677, 18719, 16724, 23, 3842, 16, 1, 2, 2, 2],\n",
              " ['Con',\n",
              "  'séqu',\n",
              "  'emment</w>',\n",
              "  'leurs</w>',\n",
              "  'codes</w>',\n",
              "  'compr',\n",
              "  'endraient</w>',\n",
              "  'des</w>',\n",
              "  'erreurs</w>',\n",
              "  '.</w>'])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4ySTnpTLexyi"
      },
      "source": [
        "#### Test encodage"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-qyWWzPl91PB",
        "outputId": "33e4a753-cf0f-4f5c-8d28-26d8870192ad"
      },
      "source": [
        "encoder = WSDEncoder(flaubert_tokenizer, flaubert_config)\n",
        "\n",
        "# test encoder\n",
        "\n",
        "#@@ erreur dans le test\n",
        "#test_sents = [\"Conséquemment\", \"leurs\", \"codes\", \"comprendraient\", \"des\", \"erreurs\", \".\"]\n",
        "test_sents = [[\"Conséquemment\", \"leurs\", \"codes\", \"comprendraient\", \"des\", \"erreurs\", \".\"]]\n",
        "                    \n",
        "# les mots target sont les occurrences de du verbe \"comprendre\"\n",
        "#test_tg_wrks = [2]\n",
        "test_tg_wrks = [3]\n",
        "\n",
        "# TODO: décommenter pour tester votre méthode encode\n",
        "tid_seqs, first_trk_of_targets = encoder.encode(test_sents, test_tg_wrks, max_length=20, verbose=True, is_split_into_words = True)\n",
        "\n",
        "print('trgs',first_trk_of_targets)\n",
        "\n",
        "for tid_seq, ft in zip(tid_seqs, first_trk_of_targets):\n",
        "  #@@ plus de traces\n",
        "  print(flaubert_tokenizer.convert_ids_to_tokens(tid_seq))\n",
        "  print(\"Len = %d target token rank = %d tid_seq = %s\" % (len(tid_seq), ft, str(tid_seq)))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trgs [6]\n",
            "['<s>', 'Con', 'séqu', 'emment</w>', 'leurs</w>', 'codes</w>', 'compr', 'endraient</w>', 'des</w>', 'erreurs</w>', '.</w>', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']\n",
            "Len = 20 target token rank = 6 tid_seq = [0, 1198, 17358, 13299, 121, 5677, 18719, 16724, 23, 3842, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "n2ER6-SXhEj2",
        "outputId": "a9ad41c7-6517-411a-8db7-60a3961b254f"
      },
      "source": [
        "encoder.tokenizer.convert_ids_to_tokens(6000)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'autorisé</w>'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YHApJ8mgexyn"
      },
      "source": [
        "### Classe WSDData: encodage complet des données asfalda"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMxpzaBhexyo"
      },
      "source": [
        "import random\n",
        "class WSDData:\n",
        "    def __init__(self, corpus_type, sentences, tg_wrks, tg_lemmas, labels, encoder, max_length=350):\n",
        "        \"\"\"\n",
        "        Inputs:\n",
        "        - corpus type string (train/dev/test/val)\n",
        "        - list of sentences (each sentence = list of word strings)\n",
        "        - list of target word ranks : one per sentence\n",
        "        - list of gold label id\n",
        "        - encoder = instance of WSDEncoder\n",
        "\n",
        "        - max_length = size of encoded sequences, in nb of bert tokens \n",
        "                      (padded / truncated via encoder.encode)\n",
        "    \n",
        "        Encodes all the data using the relevant identifiers\n",
        "        \"\"\"\n",
        "        \n",
        "        self.corpus_type = corpus_type # train / dev / test / val\n",
        "        self.size = len(sentences)\n",
        "        self.encoder = encoder\n",
        "\n",
        "        self.labels = labels       # gold label ids\n",
        "        self.sentences = sentences # list of list of word strings\n",
        "        self.tg_lemmas = tg_lemmas #list of target lemmas\n",
        "        \n",
        "        self.tid_seqs, self.first_trk_of_targets = encoder.encode(sentences, tg_wrks, max_length)\n",
        "        self.tg_lemma_indexes = [lemma2i[lemma]for lemma in self.tg_lemmas]\n",
        "\n",
        "        # sequences of token ids\n",
        "        # target token ranks\n",
        "        \n",
        "        \n",
        "\n",
        "    def shuffle(self):\n",
        "      \"\"\"\n",
        "      Rearranges all the data in a new random order\n",
        "      (sentences, tg_lemmas, tg_trks, tid_seqs, labels)\n",
        "\n",
        "      NB: ** original order is lost **\n",
        "      \"\"\"\n",
        "      z = list(zip(self.labels, self.sentences, self.tg_lemma_indexes, self.tid_seqs, self.first_trk_of_targets))\n",
        "      random.shuffle(z)\n",
        "      labels, sentences, tg_lemma_indexes, tid_seqs,first_trk_of_targets = zip(*z)\n",
        "      \n",
        "\n",
        "      \n",
        "      return labels, sentences, tg_lemma_indexes, tid_seqs,first_trk_of_targets\n",
        "\n",
        " \n",
        "\n",
        "    # production de batches de données\n",
        "    def make_batches(self, batch_size, shuffle_data=False):\n",
        "        \"\"\"\n",
        "        Returns an iterator over 3 torch tensors \n",
        "        - batch of token id sequences\n",
        "        - corresponding batch of target token ranks\n",
        "        - corresponding batch of labels for these targets\n",
        "        \"\"\"\n",
        "        # TODO\n",
        "        # (use \"yield\" function to return iterator\n",
        "        bstart = 0\n",
        "        if shuffle_data:\n",
        "          self.labels, self.sentences, self.tg_lemmas, self.tid_seqs, self.first_trk_of_targets = self.shuffle()\n",
        "        N = len(self.labels)\n",
        "        while bstart < len(self.labels):\n",
        "          bend = min(bstart+batch_size,N)\n",
        "          b_labels, b_tid_seqs, b_tg_trks, b_lemmas = self.labels[bstart:bend] , self.tid_seqs[bstart:bend] , self.first_trk_of_targets[bstart:bend], self.tg_lemma_indexes[bstart:bend] \n",
        "          assert(len(b_labels)==len(b_tid_seqs))\n",
        "          yield (b_tid_seqs, b_tg_trks, b_labels, b_lemmas)#lemmas\n",
        "        \n",
        "          bstart += batch_size\n",
        "\n",
        "       \n",
        "        \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z6sSDJMarsXA",
        "outputId": "1fe1dcb0-fac6-4a5c-f157-fc977a22c9f8"
      },
      "source": [
        "sentences.keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['dev', 'train', 'test', 'val'])"
            ]
          },
          "metadata": {},
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "J6A23zEuj3vA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m6D81AP4ye10",
        "outputId": "d16ba173-7d21-4ed8-96f1-2bf1906c2673"
      },
      "source": [
        "MAX_LENGTH = 300\n",
        "wsd_data = {}\n",
        "# key = part of the split corpus (train/test/dev/val)\n",
        "for p in sentences.keys():\n",
        "    print(\"Encodage de la partie %s ...\" % p)\n",
        "    wsd_data[p] = WSDData(p, sentences[p], tg_wrks[p], tg_lemmas[p], labels[p], \n",
        "                          encoder, max_length=MAX_LENGTH)\n",
        "    # vérif que l'encodage donne bien la bonne taille\n",
        "    for i, s in enumerate(wsd_data[p].tid_seqs):\n",
        "        if len(s) != MAX_LENGTH:\n",
        "            print(\"Size bug:\", i, s)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Encodage de la partie dev ...\n",
            "Encodage de la partie train ...\n",
            "Encodage de la partie test ...\n",
            "Encodage de la partie val ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aIw1BE1wexys"
      },
      "source": [
        "## Classe WSDClassifier : le réseau de neurones pour la WSD\n",
        "\n",
        "Architecture de base = \n",
        "- le modèle *BERT (ici FlauBERT)\n",
        "- puis une couche linéaire + softmax"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DQ-w08okgaXv"
      },
      "source": [
        "### Le réseau : architecture, propagation avant, évaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYAvfY83z0dF"
      },
      "source": [
        "class MLP(nn.Module):\n",
        "  def __init__(self,input_size,output_size,hidden_size):\n",
        "    super(MLP, self).__init__()\n",
        "    self.encoder = nn.Linear(input_size,hidden_size)\n",
        "    self.decoder = nn.Linear(hidden_size,output_size)\n",
        "    self.activation = nn.Tanh()\n",
        "  def forward(self,xinput):\n",
        "    h = self.activation(self.encoder(xinput))\n",
        "    return self.decoder(h)\n",
        "\n",
        "\n",
        "classifier_mlp_weights_lemmas = WSDClassifier(num_labels, device = 'cuda', use_mlp = True, hidden_size = 100,nbr_lemmas = len(lemma2i), lemma_embedding_size = 518, add_lemmas = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFQEpfkRexyy"
      },
      "source": [
        "\n",
        "class WSDClassifier(nn.Module):\n",
        "\n",
        "    def __init__(self, num_labels, device='cpu', bert_model_name=\"flaubert/flaubert_base_cased\", freeze_bert = True, use_mlp = False, hidden_size = 100,nbr_lemmas = len(lemma2i), lemma_embedding_size = 518, add_lemmas = False):\n",
        "        super(WSDClassifier, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.use_mlp = use_mlp\n",
        "        self.add_lemmas = add_lemmas\n",
        "                \n",
        "        # le début du réseau est un réseau de type *BERT\n",
        "        # le .to(device) déclenche la copie vers un éventuel GPU\n",
        "        self.bert_layer = AutoModel.from_pretrained(bert_model_name,\n",
        "                                                   ).to(self.device)\n",
        "        # on récupère la config pour avoir la taille des embeddings bert\n",
        "        self.bert_config = AutoConfig.from_pretrained(bert_model_name)\n",
        "        \n",
        "        if self.add_lemmas:\n",
        "          #Adding lemma information\n",
        "          self.hidden_size_bert = int(self.bert_config.hidden_size)+int(lemma_embedding_size)\n",
        "          print('lemmahidden', self.hidden_size_bert)\n",
        "        else:\n",
        "          self.hidden_size_bert = int(self.bert_config.hidden_size)\n",
        "        if add_lemmas:\n",
        "          self.lemma_embedding = nn.Embedding(nbr_lemmas, lemma_embedding_size).to(self.device)\n",
        "        \n",
        "        #print('hidden',hidden_size)\n",
        "        # TODO HERE : la suite\n",
        "        # TODO: implement option freeze_bert\n",
        "        if freeze_bert:\n",
        "          for param in self.bert_layer.parameters(): #Freezing the Bert parameters\n",
        "            param.requires_grad = False\n",
        "        if self.use_mlp:\n",
        "          self.mlp = MLP(self.hidden_size_bert,num_labels,100).to(self.device)\n",
        "\n",
        "        else:\n",
        "          self.linear = torch.nn.Linear(self.hidden_size_bert,num_labels).to(self.device)\n",
        "        self.softmax = torch.nn.LogSoftmax(dim = 1).to(self.device)\n",
        "        \n",
        "        \n",
        "\n",
        "    def forward(self, b_tid_seq, b_tg_trk, b_tg_lemma_indexes = None):\n",
        "        \"\"\"\n",
        "        Inputs: (all are tensors, on the relevant device)\n",
        "            - a batch of sentences = a batch of token id sequences \n",
        "              (as output in 'input_ids' member of tokenizer output)\n",
        "            - a batch of target token rank = for each of the sentences, \n",
        "              the rank of first token of the target word to disambiguate\n",
        "\n",
        "        Output: log_softmax scores for the whole batch (batch_size x num_labels)\n",
        "        \"\"\"\n",
        "        \"\"\"\n",
        "        self.main_part = torch.nn.Sequential(\n",
        "            self.bert_layer,\n",
        "            self.linear,\n",
        "            self.softmax)\n",
        "        \"\"\"\n",
        "        \n",
        "     \n",
        "    \n",
        "        embeddings_bert = self.bert_layer(b_tid_seq, return_dict = True).last_hidden_state.to(self.device)\n",
        "        \n",
        "       \n",
        "      \n",
        "     \n",
        "      \n",
        "        \n",
        "        embeddings_bert_tgt = embeddings_bert[torch.arange(embeddings_bert.size(0)), b_tg_trk].to(self.device) #last hidden state, ranks. We extract the \n",
        "        #necessary contextualised embeddings from the last layer\n",
        "        \n",
        "        if self.add_lemmas: #add lemma information\n",
        "          lemma_embeddings = self.lemma_embedding(b_tg_lemma_indexes).to(self.device)\n",
        "          embeddings_bert_tgt = torch.cat((embeddings_bert_tgt,lemma_embeddings), dim = 1).to(self.device)#If we use lemmas, we concat bert embeddings and information about lemmas\n",
        "        \n",
        "\n",
        "        \n",
        "        if self.use_mlp:\n",
        "          linear_tgt = self.mlp(embeddings_bert_tgt).to(self.device)\n",
        "        else:\n",
        "          linear_tgt = self.linear(embeddings_bert_tgt).to(self.device)\n",
        "        \n",
        "        soft_tgt = self.softmax(linear_tgt).to(self.device)\n",
        "       \n",
        "        \n",
        "        return soft_tgt\n",
        "        #idx = torch.tensor([1, 2])\n",
        "        #x[torch.arange(x.size(0)), idx]\n",
        "     \n",
        "        \n",
        "        # TODO HERE\n",
        "        #  - récuperer les embeddings *bert de tous les tokens des phrases du batch \n",
        "        #    [ batch_size * seq_len * bert_emb_size ]\n",
        "        #\n",
        "        #  - isoler l'embedding du token target pour toutes les phrases du batch\n",
        "        #    [ batch_size * bert_emb_size ]\n",
        "\n",
        "        #\n",
        "        #    Indications pour le faire élégamment et efficacement:\n",
        "        #    https://discuss.pytorch.org/t/how-to-select-specific-vector-in-3d-tensor-beautifully/37724\n",
        "        #\n",
        "        #  - et suite pour fine tuning\n",
        "            \n",
        "    def run_on_dataset(self, wsd_data, optimizer, batch_size=32, validation_use = False):\n",
        "        \"\"\"\n",
        "        Run classifier on wsd_data and compute accuracy\n",
        "        Inputs = \n",
        "         - wsd_data (WSDDataset instance)\n",
        "         - batch_size\n",
        "        Returns:\n",
        "         - list of predicted label ids\n",
        "        \"\"\"\n",
        "        pred_labels = []\n",
        "        val_losses = []\n",
        "        batch_acc = []\n",
        "        \n",
        "        loss_function = nn.NLLLoss()\n",
        "\n",
        "\n",
        "        \n",
        "        # toggle evaluation mode of the model (IMPORTANT)\n",
        "        self.eval()\n",
        "        for b_tid_seqs, b_tg_trks, b_labels, b_lemma_idx in wsd_data.make_batches(32, shuffle_data=False):\n",
        "          with torch.no_grad():\n",
        "            b_tid_seqs = torch.tensor(b_tid_seqs, device=self.device).to(self.device)\n",
        "            \n",
        "            b_tg_trks = torch.tensor(b_tg_trks, device=self.device).to(self.device)\n",
        "            b_labels = torch.tensor(b_labels, device=self.device).to(self.device)\n",
        "            b_lemma_idx = torch.tensor(b_lemma_idx, device=self.device).to(self.device)\n",
        "            log_probs = classifier(b_tid_seqs, b_tg_trks, b_lemma_idx).to(self.device)\n",
        "        \n",
        "            log_probs = self(b_tid_seqs, b_tg_trks,b_lemma_idx)\n",
        "            b_pred_labels = torch.argmax(log_probs, dim=1).to(self.device)\n",
        "            \n",
        "            pred_labels.extend(b_pred_labels)\n",
        "            \n",
        "            if validation_use:\n",
        "            \n",
        "              loss = loss_function(log_probs,b_labels)\n",
        "    \n",
        "              val_losses.append(loss.item())\n",
        "          \n",
        "          batch_acc.append(self.evaluate(b_labels,b_pred_labels))\n",
        "          \n",
        "\n",
        "\n",
        "        \n",
        "     \n",
        "        return pred_labels, val_losses, mean(batch_acc), b_labels, batch_acc\n",
        "\n",
        "    def evaluate(self, gold_labels, pred_labels):\n",
        "        \"\"\" returns accuracy, nb_correct, nb_total \"\"\"\n",
        "        \n",
        "        acc = float(torch.sum(gold_labels == pred_labels))/len(gold_labels)\n",
        "        return acc\n",
        "\n",
        "        # TODO \n",
        "\n",
        "\n",
        "        \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "J1DpaaQTHpQp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(i2label)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XbngZSY6sB2v",
        "outputId": "481b27a6-fd77-4e98-a3b6-09db2e6d4efa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "106"
            ]
          },
          "metadata": {},
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vrkbQ4hVexy1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "23b53112c64049559dd7df682a087a07",
            "d77bb697c63b4ea68207b2ddeac7a8ba",
            "1dc3b755a6f54a569252e71d897e6670",
            "651b2405314b43528851c65929613042",
            "bbb17e9bd31c4b92bce16c87ebcd8874",
            "ba5716e4dc774b79a87aaa8797e823c1",
            "0d2f9fe61e6d43ceac986353d6504998",
            "7cbd46cff2a440ffa108be0578cb1984",
            "866aaccdb00546d382e3c3c3b50c07d3",
            "ac78666b433b428ebc439554c2f4e5b1",
            "3f891515da404620b25a264329861323"
          ]
        },
        "outputId": "4370a86c-dc6f-40ea-e12f-f4fc48a20b22"
      },
      "source": [
        "# une instance de WSDClassifier\n",
        "num_labels = len(i2label)\n",
        "classifier = WSDClassifier(num_labels, device = 'cuda'\n",
        ")\n",
        "\n",
        "#en décommentant, on voit le nb impressionnant de paramètres du modèle *BERT ...\n",
        "for name, param in classifier.named_parameters():\n",
        "  print(\"PARAM named %s, of shape %s\" % (name, str(param.shape)))\n",
        "  print(param.requires_grad)\n",
        "    \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/553M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "23b53112c64049559dd7df682a087a07"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at flaubert/flaubert_base_cased were not used when initializing FlaubertModel: ['pred_layer.proj.weight', 'pred_layer.proj.bias']\n",
            "- This IS expected if you are initializing FlaubertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing FlaubertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PARAM named bert_layer.position_embeddings.weight, of shape torch.Size([512, 768])\n",
            "False\n",
            "PARAM named bert_layer.embeddings.weight, of shape torch.Size([68729, 768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm_emb.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm_emb.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.0.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.1.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.2.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.3.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.4.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.5.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.6.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.7.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.8.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.9.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.10.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.q_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.q_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.k_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.k_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.v_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.v_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.out_lin.weight, of shape torch.Size([768, 768])\n",
            "False\n",
            "PARAM named bert_layer.attentions.11.out_lin.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.0.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.0.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.1.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.1.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.2.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.3.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.3.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.4.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.4.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.5.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.5.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.6.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.6.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.7.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.7.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.8.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.8.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.9.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.9.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.10.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.10.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.11.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm1.11.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.0.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.0.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.0.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.0.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.1.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.1.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.1.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.1.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.2.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.2.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.2.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.2.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.3.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.3.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.3.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.3.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.4.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.4.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.4.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.4.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.5.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.5.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.5.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.5.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.6.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.6.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.6.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.6.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.7.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.7.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.7.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.7.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.8.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.8.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.8.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.8.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.9.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.9.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.9.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.9.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.10.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.10.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.10.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.10.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.11.lin1.weight, of shape torch.Size([3072, 768])\n",
            "False\n",
            "PARAM named bert_layer.ffns.11.lin1.bias, of shape torch.Size([3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.11.lin2.weight, of shape torch.Size([768, 3072])\n",
            "False\n",
            "PARAM named bert_layer.ffns.11.lin2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.0.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.0.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.1.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.1.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.2.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.2.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.3.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.3.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.4.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.4.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.5.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.5.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.6.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.6.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.7.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.7.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.8.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.8.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.9.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.9.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.10.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.10.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.11.weight, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named bert_layer.layer_norm2.11.bias, of shape torch.Size([768])\n",
            "False\n",
            "PARAM named linear.weight, of shape torch.Size([106, 768])\n",
            "True\n",
            "PARAM named linear.bias, of shape torch.Size([106])\n",
            "True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6LeXSxEXexy5"
      },
      "source": [
        "#### Test de la propagation avant"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4iIZvzDexy7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1f4f9304-78f9-4106-92ec-326e54a181c7"
      },
      "source": [
        "\n",
        "\n",
        "# inutile de calculer les gradients\n",
        "with torch.no_grad():\n",
        "    # mode evaluation et pas train\n",
        "    classifier.eval()\n",
        "    for b_tid_seqs, b_tg_trks, b_labels, _ in wsd_data['dev'].make_batches(32, shuffle_data=True):\n",
        "        b_tid_seqs = torch.tensor(b_tid_seqs, device=classifier.device)\n",
        "        b_tg_trks = torch.tensor(b_tg_trks, device=classifier.device)\n",
        "        b_labels = torch.tensor(b_labels, device=classifier.device).to(classifier.device)\n",
        "        print('input size : ',b_tid_seqs.size(),\" reference size : \",b_tg_trks.size())\n",
        "        log_probs = classifier(b_tid_seqs, b_tg_trks)\n",
        "        print('output size : ',log_probs.size(),\" reference size : \",b_labels.size())\n",
        "       \n",
        "        gold = b_labels[0] #.item()\n",
        "        print(\"GOLD LABEL of first ex %d ( = %s)\" % (gold, i2label[gold]))\n",
        "        print(\"LOG_PROBS before training: %s\\n\\n\" % str(log_probs[0]))\n",
        "        break\n",
        "        \n",
        "\n",
        "        \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input size :  torch.Size([32, 300])  reference size :  torch.Size([32])\n",
            "output size :  torch.Size([32, 106])  reference size :  torch.Size([32])\n",
            "GOLD LABEL of first ex 59 ( = FR_Statement-manner-noise)\n",
            "LOG_PROBS before training: tensor([-6.7105, -6.7867, -4.7844, -5.6924, -5.4828, -5.4459, -5.5531, -5.4943,\n",
            "        -5.9680, -5.5288, -6.1501, -4.9371, -4.3809, -4.2544, -5.0705, -6.2594,\n",
            "        -5.7332, -5.1021, -5.1539, -4.8033, -4.1711, -6.3250, -4.0623, -6.6760,\n",
            "        -4.1447, -5.3594, -3.8118, -4.6208, -5.6900, -5.9616, -4.0374, -5.2778,\n",
            "        -4.6664, -4.0034, -4.1932, -5.6759, -5.4935, -6.4562, -5.5260, -4.8584,\n",
            "        -5.9689, -5.3847, -6.5034, -4.8563, -5.3975, -4.7420, -4.6736, -4.2011,\n",
            "        -3.1192, -5.6371, -3.9862, -4.8696, -3.0066, -4.0557, -5.9870, -6.2210,\n",
            "        -4.4476, -6.5916, -5.1059, -5.3135, -5.9809, -2.8584, -5.7406, -5.3571,\n",
            "        -4.1119, -4.2763, -4.1690, -4.8643, -5.7209, -6.0943, -3.8044, -6.9562,\n",
            "        -3.5216, -4.9633, -4.1077, -5.6175, -5.2129, -4.8709, -3.6331, -4.1898,\n",
            "        -5.0117, -4.1479, -4.3558, -3.9716, -6.0206, -4.5263, -5.0439, -4.6454,\n",
            "        -4.4732, -4.5468, -5.3914, -4.8775, -3.6574, -6.2514, -5.7525, -4.7976,\n",
            "        -5.3420, -4.4159, -5.5781, -4.9510, -5.5982, -3.7842, -6.0868, -5.1554,\n",
            "        -5.7633, -4.8145])\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v2Hz7e9-oZcF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "288051f0-5cc1-4d81-f34e-742f7e2b8566"
      },
      "source": [
        "import numpy as np\n",
        "np.array(wsd_data['train'].labels).shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16792,)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "WqyFZeFdyNdQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "etG6nI8CofMp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62495e87-c285-4114-9e4d-7b9ef9397b65"
      },
      "source": [
        "np.array(list(label2i.values()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,\n",
              "        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,\n",
              "        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,\n",
              "        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,\n",
              "        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,\n",
              "        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,\n",
              "        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,\n",
              "        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,\n",
              "       104, 105])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aikfNi0zexy9"
      },
      "source": [
        "### Entraînement : fine-tuning sur la tâche de WSD"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        },
        "id": "9p28nDcdYUqi",
        "outputId": "0dadf5a9-88e4-461d-9abf-b7f870789683"
      },
      "source": [
        "# training\n",
        "from statistics import mean\n",
        "\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "LR = 0.0005\n",
        "n_epochs = 20\n",
        "import numpy as np\n",
        "stop_early = 0\n",
        "early_stopping_patience = 6\n",
        "\n",
        "\n",
        "#loss_function = nn.CrossEntropyLoss()\n",
        "loss_function = nn.NLLLoss() \n",
        "#optimizer = optim.SGD(classifier.parameters(), lr=LR)\n",
        "optimizer = optim.Adam(classifier.parameters(), lr=LR)\n",
        "\n",
        "config_name = 'sequoiaftb.asfalda_1_3.wsd.lr' + 'Adam' + str(LR) + '_bs' + str(BATCH_SIZE)\n",
        "out_model_file = './' + config_name + '.model'\n",
        "out_log_file = './' + config_name + '.log'\n",
        "\n",
        "\n",
        "# perte à chaque époque (sur le train / sur le validation set)\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "val_accs = []\n",
        "min_val_loss = None\n",
        "epoch_losses = []\n",
        "# pour tests: utiliser training sur dev\n",
        "#train_data = data['dev'] # data['train']\n",
        "train_data = wsd_data['train']\n",
        "val_data = wsd_data['val']\n",
        "dev_data = wsd_data['dev']\n",
        "classifier.train()\n",
        "early_stopping_patience = 0 #how many times dev accuracy might be smaller than previous time\n",
        "stop_early = 6\n",
        "\n",
        "print('Training.....')\n",
        "acc = 0\n",
        "for epoch in range(n_epochs):  # loop over the dataset multiple times\n",
        "  classifier.train()\n",
        "  \n",
        "  print('Training..... epoch nr: ', epoch)\n",
        "  for b_tid_seqs, b_tg_trks, b_labels, _ in train_data.make_batches(64, shuffle_data=True):\n",
        "    \n",
        "  \n",
        "    optimizer.zero_grad()\n",
        "    b_tid_seqs = torch.tensor(b_tid_seqs, device=classifier.device).to(classifier.device)\n",
        "    b_tg_trks = torch.tensor(b_tg_trks, device=classifier.device).to(classifier.device)\n",
        "    b_labels = torch.tensor(b_labels, device=classifier.device).to(classifier.device)\n",
        "    log_probs = classifier(b_tid_seqs, b_tg_trks).to(classifier.device)\n",
        "    loss = loss_function(log_probs,  b_labels)\n",
        "    #print(log_probs.shape,loss)\n",
        "    \n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_losses.append(loss.item())\n",
        "  epoch_losses.append(sum(train_losses)/len(train_losses))\n",
        "  \n",
        "  pred_labels, val_losses, val_acc, _, _ = classifier.run_on_dataset(val_data, optimizer, batch_size=32, validation_use=True)\n",
        "  print('--------')\n",
        "  print('train loss: ',epoch_losses[-1])\n",
        "  print('val accuracy: ', val_acc)\n",
        "  print('--------')\n",
        "  if val_acc>=acc:\n",
        "    acc=val_acc\n",
        "  else:\n",
        "    early_stopping_patience +=1\n",
        "    if early_stopping_patience == stop_early:\n",
        "      print('Stopping early...')\n",
        "      break\n",
        "\n",
        "  \n",
        "    \n",
        "  \n",
        " \n",
        "\n",
        "\n",
        "        \n",
        "  \n",
        "           \n",
        "# TODO HERE\n",
        "# training\n",
        "# - basic : train for NB_EPOCHS +\n",
        "# - BONUS : early stopping: stop epoch loop as soon as accuracy on dev decreases+\n",
        "\n",
        "# don't forget to toggle \n",
        "# - classifier.train() when training on train\n",
        "# - classifier.eval() when evaluating on val corpus\n",
        "\n",
        "\n",
        "print(\"train losses: %s\" % ' / '.join([ \"%.4f\" % x for x in epoch_losses]))\n",
        "print(\"val   losses: %s\" % ' / '.join([ \"%.4f\" % x for x in val_losses]))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training.....\n",
            "Training..... epoch nr:  0\n",
            "--------\n",
            "train loss:  1.2358378282971256\n",
            "val accuracy:  0.7877236346516008\n",
            "--------\n",
            "Training..... epoch nr:  1\n",
            "--------\n",
            "train loss:  1.0202058154253906\n",
            "val accuracy:  0.8123234463276836\n",
            "--------\n",
            "Training..... epoch nr:  2\n",
            "--------\n",
            "train loss:  0.9100279513042991\n",
            "val accuracy:  0.8205626177024482\n",
            "--------\n",
            "Training..... epoch nr:  3\n",
            "--------\n",
            "train loss:  0.8397154466677528\n",
            "val accuracy:  0.8263888888888888\n",
            "--------\n",
            "Training..... epoch nr:  4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(epoch_losses)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()\n",
        "plt.plot(val_accs)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "d7htoPi1FHa2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aOvw3cXaYKtz"
      },
      "source": [
        "Below is a version of classifier with added weights"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.array(wsd_data['train'].labels).shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lYw0o-u5-UXP",
        "outputId": "f8aeeded-04d8-4667-d764-ffe6f83f0f88"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16792,)"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = wsd_data['train']\n",
        "counter=0\n",
        "for b_tid_seqs, b_tg_trks, b_labels, _ in train_data.make_batches(64, shuffle_data=True):\n",
        "  counter+=1\n",
        "  pass\n",
        "print(\"b_tid_seqs :\" , b_tid_seqs,\"\\nb_tg_trks\", b_tg_trks, \"\\nb_labels: \" ,b_labels)\n",
        "print(\"b_tid_seqs :\" , len([i for i in b_tid_seqs[0]]),\"\\nb_tg_trks\", len(b_tg_trks), \"\\nb_labels: \" ,len( b_labels))\n",
        "counter,((counter-1)*64)+24"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hxhtkXEC6cxV",
        "outputId": "5701d5ba-60e7-4d46-c433-4afd58d5793c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "b_tid_seqs : ([0, 804, 14, 21, 12311, 15, 4307, 15, 23907, 33, 20, 8739, 24, 11686, 50, 21, 9770, 100, 22, 27601, 15, 6952, 18, 24, 10412, 432, 49, 7400, 18258, 93, 21, 58, 49, 9275, 14, 72, 20, 1249, 17, 35457, 14, 32, 1367, 108, 23, 24348, 24, 297, 222, 62, 23962, 218, 19, 21, 4347, 29838, 548, 251, 126, 15, 90, 112, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 697, 22, 5049, 10110, 15, 21, 2687, 14, 182, 2668, 15, 1851, 24, 25075, 46273, 14, 115, 412, 1364, 383, 231, 14, 52, 1857, 35084, 25, 447, 15, 21, 21530, 14, 2462, 15, 289, 3869, 14, 15, 3027, 18, 15, 1366, 25588, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 55, 291, 23, 4505, 34, 69, 14, 33, 21, 273, 14, 493, 24655, 14, 22, 40964, 15, 21, 1360, 25070, 17, 1114, 18141, 38, 7974, 46355, 31, 21, 1073, 15, 21, 2313, 24, 14616, 14, 33, 26, 1143, 17429, 14, 19, 17, 1031, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 60, 319, 53, 27, 5904, 47, 37088, 893, 34, 69, 1049, 21, 3620, 23, 237, 15, 706, 43, 17, 2278, 3737, 23, 237, 9460, 31, 45, 196, 14, 36, 258, 24, 73, 39424, 72, 30, 265, 1423, 15, 6588, 18, 30, 265, 2692, 92, 15, 17, 696, 28, 7333, 14, 34, 29996, 17, 5352, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 4110, 52, 17056, 19, 177, 23, 1878, 5028, 2522, 14, 72, 17, 1405, 15, 700, 14, 378, 108, 23, 3791, 57, 436, 25, 643, 14, 35, 34, 1886, 43172, 218, 21, 212, 4031, 4461, 15, 17, 1405, 23, 550, 24, 715, 13705, 1183, 978, 70, 6681, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 320, 22240, 9070, 8093, 960, 23854, 26, 966, 29, 17146, 3900, 29, 15, 17, 774, 7025, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 174, 319, 2112, 19, 49, 1201, 45, 46, 41, 66, 34, 15, 722, 33, 30, 1836, 307, 8644, 1094, 14, 38, 22, 16946, 15, 17, 871, 18, 15, 17, 776, 14, 19, 17, 223, 3959, 28, 29, 7333, 630, 29, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 326, 11145, 9845, 4864, 48, 15931, 42, 31, 634, 215, 28, 8014, 15, 17, 14009, 3400, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 1160, 3979, 15, 18423, 14, 44, 3963, 14, 230, 33, 21, 1347, 28, 287, 14, 65, 6480, 19, 73, 44, 519, 14, 44, 33308, 14, 44, 23748, 19, 431, 23, 6135, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 154, 605, 18, 2897, 537, 14, 79, 27, 20, 119, 46, 41, 556, 4816, 19, 5998, 2095, 19782, 14, 9472, 10600, 10398, 200, 38010, 4589, 14, 21, 423, 25, 25506, 4191, 15, 17, 1805, 15, 17, 17834, 23, 210, 15, 7235, 8806, 14, 31, 9541, 33, 21, 4662, 28, 2533, 5353, 15, 17, 680, 14, 47, 54, 6873, 24386, 14, 37, 738, 269, 11415, 11541, 3685, 24, 733, 25, 836, 15, 4128, 1242, 15, 485, 14, 3772, 19, 17, 3017, 28, 2328, 25, 940, 37, 20, 2568, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 123, 17, 800, 23, 3513, 15, 234, 9236, 5141, 184, 19, 610, 100, 5395, 2160, 18, 121, 16900, 70, 2522, 50, 4359, 70, 404, 11925, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 1179, 4979, 1416, 167, 23, 29, 2812, 148, 6072, 29, 46, 35641, 36879, 15, 2925, 31, 26, 1187, 1077, 15, 130, 14, 153, 4391, 15, 3601, 232, 15, 40604, 217, 20, 235, 118, 17, 8607, 15, 60, 8993, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 129, 1572, 28, 5995, 14, 19, 568, 2184, 14, 41, 178, 24, 653, 2925, 23, 1452, 4005, 14, 46, 58, 51, 34, 42, 18, 46, 58, 20255, 14, 19, 521, 14, 2110, 108, 3088, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 156, 20, 85, 119, 14, 37, 20, 3588, 1368, 14, 22, 137, 70, 3047, 15, 21, 905, 18, 15, 17, 6271, 31624, 3950, 321, 18, 22, 440, 4736, 15, 21, 1073, 16388, 6043, 62, 4966, 19, 20109, 22, 5824, 4449, 14, 36, 271, 111, 22, 2324, 70, 11283, 404, 236, 406, 24, 26, 6238, 1368, 31, 9721, 26, 384, 8950, 117, 8458, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 209, 449, 12095, 62, 980, 17, 460, 138, 17, 3947, 28, 1126, 43, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 107, 176, 5856, 32, 17, 281, 25, 874, 24, 4941, 2055, 27, 730, 31, 2093, 17, 387, 23, 1009, 14, 63, 92, 31, 328, 88, 1288, 20, 10024, 15, 3737, 33, 67, 421, 14, 45, 32, 65, 484, 3088, 25, 1137, 15, 88, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 133, 11699, 70, 66, 1237, 43, 347, 21, 5771, 15, 17, 1939, 14, 22, 4658, 15, 17, 6025, 4162, 394, 1078, 136, 44, 36319, 31, 22, 5475, 7311, 32, 844, 23, 127, 27335, 6352, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 55, 604, 2168, 15, 17, 1415, 501, 37, 29, 20, 277, 28, 384, 924, 23, 300, 35938, 29, 14, 377, 15, 170, 31, 20, 1467, 19, 926, 14, 625, 24, 73, 1556, 396, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 64, 9324, 14949, 92, 17, 149, 1740, 50, 3911, 23, 5989, 43, 29, 190, 10757, 22, 1879, 24290, 25, 4587, 29, 14, 29, 34, 7196, 19, 17, 7306, 15, 179, 15983, 15, 71, 755, 29, 14, 29, 12552, 15, 17, 2637, 908, 10678, 14769, 7714, 1948, 29, 14, 29, 134, 74, 231, 33, 30, 46695, 12684, 12903, 29, 14, 29, 713, 3394, 3153, 60, 8747, 200, 99, 29, 14, 29, 134, 5754, 22, 31797, 18, 22, 10978, 15, 2391, 8384, 14, 15, 17378, 18, 15, 43097, 29, 14, 29, 11273, 22942, 29, 14, 29, 34, 30, 408, 95, 17637, 2177, 29, 14, 29, 395, 2554, 1452, 2440, 3545, 47, 30, 408, 3684, 29, 14, 29, 134, 1224, 17, 17671, 28, 4629, 19, 54, 1605, 38, 4335, 8334, 29, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 64, 1215, 28, 2328, 13338, 81, 35538, 758, 118, 17, 5319, 24, 30, 23, 538, 13006, 33, 20, 454, 15, 1023, 1180, 38, 20, 316, 25, 586, 239, 43, 41, 53, 444, 15, 17, 1114, 2039, 57, 3362, 20146, 15, 17237, 184, 19, 310, 108, 28, 1187, 15, 81, 1724, 40, 33, 20, 1797, 39, 142, 613, 476, 40, 7888, 36, 10782, 14, 19, 17, 927, 15, 17, 2389, 18, 36, 1831, 28, 11851, 39, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 113, 27, 30, 6283, 15, 25196, 18648, 9783, 7357, 226, 20685, 7440, 6179, 1246, 6659, 8745, 14, 35, 4096, 26, 1834, 24, 611, 15, 216, 878, 15, 877, 40, 3468, 878, 15, 3601, 39, 18, 7670, 534, 383, 1767, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 64, 4275, 15, 14401, 14, 50, 493, 22, 3748, 15, 3601, 38, 17, 2619, 15, 143, 14, 53, 16562, 14, 3144, 14, 19, 26, 2307, 203, 126, 1089, 32, 204, 15, 17, 234, 964, 15, 657, 40, 207, 404, 1633, 7787, 878, 15, 3601, 39, 14, 63, 14, 19, 21, 605, 1833, 14, 207, 2613, 6620, 22, 7267, 878, 15, 3601, 14, 45, 35, 27, 203, 44, 15643, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 141, 976, 32, 20, 1131, 38, 70, 6995, 22, 3113, 146, 10587, 3488, 75, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 123, 343, 48, 45823, 19, 2595, 25, 513, 22, 13882, 1106, 15, 17, 25037, 889, 16, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]) \n",
            "b_tg_trks (21, 26, 39, 37, 32, 6, 16, 12, 17, 44, 5, 27, 2, 39, 12, 22, 15, 32, 3, 22, 19, 9, 5, 8) \n",
            "b_labels:  (45, 17, 49, 43, 17, 96, 6, 65, 5, 40, 105, 12, 87, 94, 13, 40, 40, 17, 59, 59, 40, 87, 99, 40)\n",
            "b_tid_seqs : 300 \n",
            "b_tg_trks 24 \n",
            "b_labels:  24\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(263, 16792)"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eOKrCRPFexy-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d5ab60b-142b-4436-c5e9-9831617ee55c"
      },
      "source": [
        "# training\n",
        "from statistics import mean\n",
        "import numpy as np\n",
        "stop_early = 0\n",
        "early_stopping_patience = 6\n",
        "\n",
        "\n",
        "classifier_weights = WSDClassifier(num_labels, device = 'cuda', use_mlp = False, hidden_size = 100,nbr_lemmas = len(lemma2i), lemma_embedding_size = 518, add_lemmas = False)\n",
        "BATCH_SIZE = 32\n",
        "LR = 0.0005\n",
        "n_epochs = 20\n",
        "import numpy as np\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "class_weight = compute_class_weight(\"balanced\", np.unique(list(label2i.values())),list(label2i.values()))\n",
        "\n",
        "class_weight = torch.tensor(class_weight, dtype=torch.float).to(classifier.device)\n",
        "\n",
        "#loss_function = nn.CrossEntropyLoss()\n",
        "loss_function = nn.NLLLoss(weight = class_weight) \n",
        "#optimizer = optim.SGD(classifier.parameters(), lr=LR)\n",
        "optimizer = optim.Adam(classifier_weights.parameters(), lr=LR)\n",
        "\n",
        "config_name = 'sequoiaftb.asfalda_1_3.wsd.lr' + 'Adam' + str(LR) + '_bs' + str(BATCH_SIZE)\n",
        "out_model_file = './' + config_name + '.model'\n",
        "out_log_file = './' + config_name + '.log'\n",
        "\n",
        "\n",
        "# perte à chaque époque (sur le train / sur le validation set)\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "val_accs = []\n",
        "min_val_loss = None\n",
        "epoch_losses = []\n",
        "# pour tests: utiliser training sur dev\n",
        "#train_data = data['dev'] # data['train']\n",
        "train_data = wsd_data['train']\n",
        "val_data = wsd_data['val']\n",
        "dev_data = wsd_data['dev']\n",
        "classifier_weights.train()\n",
        "early_stopping_patience = 0 #how many times dev accuracy might be smaller than previous time\n",
        "stop_early = 6\n",
        "\n",
        "print('Training.....')\n",
        "acc = 0\n",
        "for epoch in range(n_epochs):  # loop over the dataset multiple times\n",
        "  classifier_weights.train()\n",
        "  \n",
        "  print('Training..... epoch nr: ', epoch)\n",
        "  for b_tid_seqs, b_tg_trks, b_labels, _ in train_data.make_batches(64, shuffle_data=True):\n",
        "    \n",
        "  \n",
        "    optimizer.zero_grad()\n",
        "    b_tid_seqs = torch.tensor(b_tid_seqs, device=classifier_weights.device).to(classifier_weights.device)\n",
        "    b_tg_trks = torch.tensor(b_tg_trks, device=classifier_weights.device).to(classifier_weights.device)\n",
        "    b_labels = torch.tensor(b_labels, device=classifier_weights.device).to(classifier_weights.device)\n",
        "    log_probs = classifier_weights(b_tid_seqs, b_tg_trks).to(classifier_weights.device)\n",
        "  \n",
        "    loss = loss_function(log_probs,  b_labels)\n",
        "    \n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_losses.append(loss.item())\n",
        "  epoch_losses.append(sum(train_losses)/len(train_losses))\n",
        " \n",
        "  \n",
        "  pred_labels, val_losses, val_acc, _, _ = classifier_weights.run_on_dataset(val_data, optimizer, batch_size=32, validation_use=True)\n",
        "  val_accs.append(val_acc)\n",
        "  print('--------')\n",
        "  print('train loss: ',epoch_losses[-1])\n",
        "  print('val accuracy: ', val_acc)\n",
        "  print('--------')\n",
        " \n",
        "  if val_acc>=acc:\n",
        "    acc=val_acc\n",
        "  else:\n",
        "    early_stopping_patience +=1\n",
        "    if early_stopping_patience == stop_early:\n",
        "      print('Stopping early...')\n",
        "      break\n",
        "\n",
        "  \n",
        "    \n",
        "  \n",
        " \n",
        "\n",
        "\n",
        "        \n",
        "  \n",
        "           \n",
        "# TODO HERE\n",
        "# training\n",
        "# - basic : train for NB_EPOCHS +\n",
        "# - BONUS : early stopping: stop epoch loop as soon as accuracy on dev decreases+\n",
        "\n",
        "# don't forget to toggle \n",
        "# - classifier.train() when training on train\n",
        "# - classifier.eval() when evaluating on val corpus\n",
        "\n",
        "\n",
        "print(\"train losses: %s\" % ' / '.join([ \"%.4f\" % x for x in epoch_losses]))\n",
        "print(\"val   losses: %s\" % ' / '.join([ \"%.4f\" % x for x in val_losses]))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training.....\n",
            "Training..... epoch nr:  0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:68: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "train loss:  1.8994931034715457\n",
            "--------\n",
            "Training..... epoch nr:  1\n",
            "train loss:  1.4174946367513996\n",
            "--------\n",
            "Training..... epoch nr:  2\n",
            "train loss:  1.1948242937657316\n",
            "--------\n",
            "Training..... epoch nr:  3\n",
            "train loss:  1.0638778495527945\n",
            "--------\n",
            "Training..... epoch nr:  4\n",
            "train loss:  0.9740305513030223\n",
            "--------\n",
            "Training..... epoch nr:  5\n",
            "train loss:  0.9094929509006525\n",
            "--------\n",
            "Training..... epoch nr:  6\n",
            "train loss:  0.8593045305875512\n",
            "--------\n",
            "Training..... epoch nr:  7\n",
            "train loss:  0.8188153217783899\n",
            "--------\n",
            "Training..... epoch nr:  8\n",
            "train loss:  0.7860050055363231\n",
            "--------\n",
            "Training..... epoch nr:  9\n",
            "train loss:  0.7582240884521615\n",
            "--------\n",
            "Training..... epoch nr:  10\n",
            "train loss:  0.7352795007986632\n",
            "--------\n",
            "Training..... epoch nr:  11\n",
            "train loss:  0.7148991766018635\n",
            "--------\n",
            "Training..... epoch nr:  12\n",
            "train loss:  0.6974163156184721\n",
            "--------\n",
            "Training..... epoch nr:  13\n",
            "train loss:  0.6811963767339392\n",
            "--------\n",
            "Training..... epoch nr:  14\n",
            "train loss:  0.6667022430738115\n",
            "--------\n",
            "Stopping early...\n",
            "train losses: 1.8995 / 1.4175 / 1.1948 / 1.0639 / 0.9740 / 0.9095 / 0.8593 / 0.8188 / 0.7860 / 0.7582 / 0.7353 / 0.7149 / 0.6974 / 0.6812 / 0.6667\n",
            "val   losses: 0.4717 / 0.4840 / 0.5912 / 0.4842 / 0.4176 / 0.5760 / 0.3416 / 0.8078 / 0.3231 / 0.5183 / 0.8220 / 0.1731 / 0.3206 / 0.2864 / 0.2439 / 0.7306 / 0.2595 / 0.5904 / 0.2447 / 0.6989 / 0.3896 / 0.5526 / 0.2932 / 0.3206 / 0.1481 / 0.3572 / 0.2909 / 0.2843 / 0.6300 / 0.7361 / 0.8685 / 0.4311 / 0.5826 / 0.8282 / 0.6725 / 0.2721 / 0.7895 / 0.3393 / 0.2945 / 0.1981 / 0.6431 / 0.6565 / 0.5096 / 0.7315 / 0.2652 / 0.8269 / 0.2558 / 0.3350 / 0.8115 / 0.9166 / 0.7100 / 0.5071 / 0.4738 / 0.6579 / 0.4897 / 1.0015 / 0.5452 / 1.1962 / 1.2202\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "LIXDclLpntmM",
        "outputId": "7a67fa45-3617-4ee4-a3b1-24204c586160"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(epoch_losses)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3RdZZ3/8ff35H5r7s1Jm6RpS5u06VViL6DQAtJwUcRRmaLgOCirgsq4dEbnJzPj+rl+Dt7QGR1wKjAVBlFELo5KW3CAKi2UFFp6L703ba5N0yRt0uby/P44pyFAr0lOds7Zn9daWU3O3j3nc1bTfPLsvZ9nm3MOERHxr4DXAURExFsqAhERn1MRiIj4nIpARMTnVAQiIj4X73WAC5WXl+dKS0u9jiEiElXWrVvX5JzLP922qCuC0tJSqqurvY4hIhJVzGzfmbbp0JCIiM+pCEREfE5FICLicyoCERGfUxGIiPicikBExOdUBCIiPuebInirvo1v/34LJ7p7vI4iIjKi+KYIao508OBf9rB612Gvo4iIjCi+KYJLLsolPSmeFZvqvI4iIjKi+KYIkuLjuKJ8NM9tqaenV3dlExE5xTdFAFA1LcjhYyep3tvsdRQRkRHDV0Vw+eR8kuIDLN+sw0MiIqf4qgjSkuL54KR8VmyqwzkdHhIRAZ8VAYQODx062snGg0e9jiIiMiL4rgiumjKauICxXFcPiYgAPiyCrNRE5k/IZbkOD4mIAD4sAoBF04LsbjrGzoZ2r6OIiHjOn0UwtQAzWKGrh0RE/FkEo0cl876SbF1GKiJCBIvAzB4yswYz23SG7Zlm9j9mtsHMNpvZZyOV5XQWVRSw6WArB5qPD+fLioiMOJEcESwDqs6y/U5gi3NuJrAA+KGZJUYwzzssqggCOjwkIhKxInDOrQLOtpaDAzLMzID08L7dkcrzbuNy05hSOEpFICK+5+U5gp8CU4BDwEbgLudc7+l2NLPbzazazKobGxuHLEBVRZDqfUdoaOscsucUEYk2XhbBImA9MAaYBfzUzEadbkfn3FLnXKVzrjI/P3/IAlRNC+IcPLelfsieU0Qk2nhZBJ8FnnQhO4E9QPlwBphckM74vDRWbFYRiIh/eVkE+4ErAcysACgDdg9nADNjUUWQ1TubONrRNZwvLSIyYkTy8tHHgDVAmZnVmNltZrbEzJaEd/k2cImZbQT+BHzdOdcUqTxnsqiigO5ex/9u06hARPwpPlJP7JxbfI7th4CrI/X652tmURbBUcks31THjbOLvI4jIjLsfDmzuL9AwFhUUcBLOxo5fnLYrl4VERkxfF8EEFqErrOrl1U7hu7SVBGRaKEiAOaU5pCdmqB7FIiIL6kIgPi4AB+aWsCftjVwsvu0c9pERGKWiiBsUUWQts5u1uw+7HUUEZFhpSIIu/SiPNIS43R4SER8R0UQlpwQx8Ly0Ty3pY6eXt3CUkT8Q0XQT9W0IE3tJ1m374jXUUREho2KoJ8FZaNJjA/o8JCI+IqKoJ/0pHgum5THis11OKfDQyLiDyqCd1lUEeRgSwebDrZ6HUVEZFioCN7lqikFxAVMdy4TEd9QEbxLdloic8fnsFxFICI+oSI4jappQXY2tLOzoc3rKCIiEaciOI2rpwYBdOcyEfEFFcFpBDOTmV2SpctIRcQXVARnUFURZOPBo9QcOe51FBGRiFIRnMGiCh0eEhF/UBGcQWleGuXBDF1GKiIxT0VwFosqgry2t5nGthNeRxERiRgVwVlUTQviHDy/VYeHRCR2qQjOojyYwbjcVF09JCIxTUVwFmZGVUWQ1buaONrR5XUcEZGIUBGcw6JpQbp6HC9sa/A6iohIRESsCMzsITNrMLNNZ9lngZmtN7PNZvZSpLIMxqyiLEZnJOnwkIjErEiOCJYBVWfaaGZZwH3AR5xzFcAnIphlwAIBY1FFkJd2NNJxssfrOCIiQy5iReCcWwU0n2WXm4EnnXP7w/uP2GMvVdOCdHT1sOqtRq+jiIgMOS/PEUwGss3sRTNbZ2a3nmlHM7vdzKrNrLqxcfh/GM8Zn0NWagIrdHhIRGKQl0UQD1wMXAcsAv7JzCafbkfn3FLnXKVzrjI/P384MwKQEBfgqikFPL+1npPdvcP++iIikeRlEdQAK5xzx5xzTcAqYKaHec6qqiJIa2c3r+w+7HUUEZEh5WURPAN8wMzizSwVmAts9TDPWX1gUh6piXG6c5mIxJxIXj76GLAGKDOzGjO7zcyWmNkSAOfcVmA58CawFnjAOXfGS029lpwQx8Ky0azcXE9Pr/M6jojIkImP1BM75xafxz7fB74fqQxDbdG0IH/YWMvr+4/w/tIcr+OIiAwJzSy+AAvL8kmMC+jqIRGJKSqCC5CRnMAHJuWxfHMdzunwkIjEBhXBBaqqCFJzpIPNh1q9jiIiMiRUBBfoqqkFBAzduUxEYoaK4ALlpCUyd3yuFqETkZihIhiARRUFvNXQzs6Gdq+jiIgMmopgAK6uCAI6PCQisUFFMABjslKYWZzFShWBiMQAFcEAVVUE2VBzlIMtHV5HEREZFBXBAC2qKADQqEBEop6KYIAm5KdTVpChq4dEJOqpCAZh0bQgr+1tpqn9hNdRREQGTEUwCIsqCuh18PyWeq+jiIgMmIpgEKYWjqI4J0X3KBCRqKYiGAQzo6oiyOqdh2nt7PI6jojIgKgIBqlqWpCTPb28sK3B6ygiIgOiIhik2cXZ5GckaZaxiEQtFcEgBQLGoooCXtjWSGdXj9dxREQumIpgCCyqCNLR1cOqHY1eRxERuWAqgiEwb0Iuo5LjdfWQiEQlFcEQSIgLcNXUAp7fUk9XT6/XcURELoiKYIhUVQRp7ezm1d3NXkcREbkgKoIhctnkfFIS4li+udbrKCIiF0RFMESSE+JYWJ7Pis319PY6r+OIiJy3iBWBmT1kZg1mtukc+73fzLrN7OORyjJcFlUEaWw7wRsHjngdRUTkvEVyRLAMqDrbDmYWB3wXWBnBHMPmivLRJMYHeOgve72OIiJy3iJWBM65VcC5zpx+CfgtEBPrM2QkJ/DlKy7iDxtreWb9Qa/jiIicF8/OEZjZWOBG4P7z2Pd2M6s2s+rGxpE9aWvJ5RO5eFw2dz+9SbexFJGo4OXJ4h8DX3fOnfPCe+fcUudcpXOuMj8/fxiiDVx8XIAffXIWvb2Orz2+QSeORWTE87IIKoFfmdle4OPAfWb2UQ/zDJmS3FT+5cMVrNl9mAf/ssfrOCIiZ+VZETjnxjvnSp1zpcATwB3Ouae9yjPUPlFZxNVTC/j+iu1srW31Oo6IyBlF8vLRx4A1QJmZ1ZjZbWa2xMyWROo1RxIz418/Np1RKQl85dfrtTKpiIxY8ZF6Yufc4gvY928ilcNLuelJfP/jM/jsstf44crtfPO6qV5HEhF5D80sjrCF5aP59LwSHvjLHlbvavI6jojIe6gIhsE3r53K+Nw0vvb4Bo526N7GIjKyqAiGQUpiHD+6aRYNbSf452fOuuKGiMiwUxEMk5nFWXz5ykk8s/4Qv9twyOs4IiJ9VATD6I4FE5ldksXdT23kkGYdi8gIoSIYRvFxAX580yy6ex1f+41mHYvIyKAiGGbjctP45+unsnrXYR56WbOORcR7KgIP3PT+Yq6aUsD3Vmxne12b13FExOdUBB4wM+75q+mMSo7nrl+9wYluzToWEe+ctQjMLNPM7jGzbWbWbGaHzWxr+LGs4QoZi/LSk/jex2ewra6Ne1fu8DqOiPjYuUYEjwNHgAXOuRznXC6wMPzY45EOF+uuKC/g5rklLP3zbl7ZfdjrOCLiU+cqglLn3Hedc3WnHnDO1TnnvguMi2w0f7j7uimU5qbx1cc30NqpWcciMvzOVQT7zOwfzKzg1ANmVmBmXwcORDaaP6QmxvOjm2ZR19rJvzyz2es4IuJD5yqCm4Bc4CUzO2JmzcCLQA7wyQhn841ZxVl86YqLeOqNg/z+Tc06FpHhddYicM4dAf4L+CJQHD5PMMU593VgznAE9IsvLryIWcVZfPOpTdQd7fQ6joj4yLmuGvoy8AyhIthkZjf02/ydSAbzm/i4AD+6aRYnu3s161hEhtW5Dg19HrjYOfdRYAHwT2Z2V3ibRTKYH43PS+Ofrp/KX3Y2sWz1Xq/jiIhPnKsIAs65dgDn3F5CZXCNmd2LiiAiFs8p5sry0dyzfJtmHYvIsDhXEdSb2axTX4RL4XogD5geyWB+FZp1PIOMpHj+7tfrNetYRCLuXEVwK1DX/wHnXLdz7lbgsoil8rn8jCS++1cz2Frbyr3PadaxiETWua4aquk/mexd216OTCQBuGpqAYvnFLN0lWYdi0hkadG5Eezu66YyLidVs45FJKJUBCNYWtLbs46/pVnHIhIhKoIRbnZJNl9ceBFPvnGQP7xZ63UcEYlBESsCM3vIzBrMbNMZtn/KzN40s41mttrMZkYqS7T74hUXMbM4i//z1EbNOhaRIRfJEcEyoOos2/cAlzvnpgPfBpZGMEtUSwjf6/hkdy9//4RmHYvI0IpYETjnVgHNZ9m+OryWEcArQFGkssSC8Xlp3H39FP78VhO/WLPX6zgiEkNGyjmC24Bnz7TRzG43s2ozq25sbBzGWCPLzXNKQrOOn93GW/WadSwiQ8PzIjCzhYSK4Otn2sc5t9Q5V+mcq8zPzx++cCPMqVnH6Unx3PHo69S36nyBiAyep0VgZjOAB4AbnHOaNXUe8jOS+Mni2Rxq6eCj//Ey2+pavY4kIlHOsyIwsxLgSeAW55zWUbgAl1yUx+NL5tPrHB+/fw2rdvj3cJmIDF4kLx99DFgDlJlZjZndZmZLzGxJeJd/JnT3s/vMbL2ZVUcqSyyqGJPJ03deSlF2Cp9d9hq/fm2/15FEJEqZc9F1KWJlZaWrrlZnnNLW2cWdv3yDVTsauXPhRL76oTICAa0QLiLvZGbrnHOVp9vm+cliGZyM5AQe/Ewli+cU8x8v7NLS1SJyweK9DiCDlxAX4Ds3TqckJ43vLt9G3dFO/vOWi8lOS/Q6mohEAY0IYoSZ8YUFE/nJ4tmsr2nhY/evZt/hY17HEpEooCKIMR+eOYZffm4uLcdPcuN9q1m378i5/5KI+JqKIAZVlubw5B2XMio5nsU/f4U/btSqpSJyZiqCGDU+L40n77iU6WMzuePR1/nPl3YRbVeIicjwUBHEsJy0RB793Fyum1HIvz67jbuf3kR3T6/XsURkhNFVQzEuOSGOn/z1bIqzU/nZS7s41NLBT25+H+lJ+qcXkRCNCHwgEDC+cU0537lxOqveauKTP1ujG9yISB8VgY/cPLeEBz9Tyb7Dx7jxvpfZWqsF60REReA7C8pG85sll+AcfOJna3hJC9aJ+J6KwIemjhnFU3deQnFOKn+77DV+tVYL1on4mYrApwozU/jNkvl84KI8vvHkRr63fJvuhSziUyoCH0tPiufBz1Ry89wS7ntxF3f9ej2dXVqwTsRvdA2hz8XHBfh/H51GSU4q9zy7jbqjHSy9pVIL1on4iEYEgpmx5PKJ/PTm2WyoOcrH7l/N3iYtWCfiFyoC6XP9jDE89vnQgnUfu3816/Y1ex1JRIaBikDe4eJxOTx1x6VkpiSw+Oev8pM/vaXzBiIxTkUg71Gal8aTX7iEK8tH88PndnDVvS/x7MZaLVonEqNUBHJa2WmJ3P/pi/nl5+eSnhTPFx59nZt//qpmI4vEIBWBnNUlE/P4/Zc+wLc/Oo2tda1c9+9/5u6nN3Lk2Emvo4nIEFERyDnFxwW4Zd44XvzaAm6dX8pjaw+w4AcvsuzlPXRpWWuRqKcikPOWlZrItz5SwbN3fZDpYzP51v9s4dp/+zN/eavJ62giMggqArlgkwsyeOS2OSy95WJOdPfy6Qdf5fMPV7PvsOYeiESjiBWBmT1kZg1mtukM283M/t3MdprZm2b2vkhlkaFnZlxdEWTlVy7j7xeV8fLOJj507yq+u3wb7Se6vY4nIhcgkiOCZUDVWbZfA0wKf9wO3B/BLBIhyQlx3LnwIl742gKun1nI/S/u4oofvMhv19VoETuRKBGxInDOrQLONjX1BuBhF/IKkGVmhZHKI5FVMCqZez85iyfvuITCrBS++psNfOz+1byx/4jX0UTkHLw8RzAWONDv65rwY+9hZrebWbWZVTc26kYqI9n7SrJ56guX8INPzORgSwc33rearz6+gYZW3RpTZKSKipPFzrmlzrlK51xlfn6+13HkHAIB4+MXF/HC1xaw5PKJ/M+GQyz8wYvc9+JOTnRruQqRkcbLIjgIFPf7uij8mMSI9KR4vnFNOSu/chnzJ+bxveXbufpHq1i5uU7LVYiMIF4Wwe+AW8NXD80Djjrnaj3MIxFSmpfGA5+p5OG/nUNCXIDbH1nHrQ+t5a36Nq+jiQhgkfrNzMweAxYAeUA98C9AAoBz7mdmZsBPCV1ZdBz4rHOu+lzPW1lZ6aqrz7mbjFBdPb389yv7+NFzOzh2sodb5o3jrisn6UY4IhFmZuucc5Wn3RZtQ3QVQWw43H6Ce5/bwWNr95MQF+DDM8dw6/xxzCjK8jqaSExSEciItaO+jV+s3stTbxzk+MkeZhZlcsv8Uq6fUUhyQpzX8URihopARrzWzi6eev0gj7yyj50N7WSlJvDJymI+PXccJbmpXscTiXoqAokazjnW7D7MI2v2sXJLPb3OsWByPrfMH8flk0cTFzCvI4pEJRWBRKW6o538cu1+Hlu7n8a2ExTnpPCpueP4ZGUxOTq5LHJBVAQS1bp6elm5uZ6H1+zl1T3NJMYHuH5GIbfOL2VmUSahC9BE5GxUBBIzdtS38ciafTz5eg3HTvYwfWwmt8wbx4dnjiElUSeXRc5ERSAxp/1EN0+9XsMjr+xjR307mSkJfOLiIj49bxyleWlexxMZcVQEErOcc6zd08zDr+xjxaY6unsdl03O59Z541hYrpPLIqecrQjihzuMyFAyM+ZOyGXuhFwaWjt5bO0Bfrl2H597uJqxWSl8al4JN1UWk5ue5HVUkRFLIwKJOV09vTy/pZ5HXtnH6l2HSYwLUDUtyPUzCrlscr4mqokvaUQgvpIQF+Ca6YVcM72QnQ2hk8vPbDjE7zYcIi0xjiumFHDd9CCXTx6tE8wiaEQgPtHV08sruw/zx411rNhcR/Oxk6QkxHFF+WiumR5kYdlo0pL0e5HELp0sFumnu6eXtXub+ePGWpZvqqep/QRJ8QEWlOVz7fRCrigfTUZygtcxRYaUikDkDHp6HdV7m3l2Ux3PbqqlvvUEifEBLpuUz7XTg1w5pYDMFJWCRD8Vgch56O11vHHgCH94M1QKtUc7SYgzPnBRHtdML+TqqQVkpWppC4lOKgKRC9Tb69hQ08Kzm+r448Zaao50EB8w5k/M5brphVxdEdR6RxJVVAQig+CcY9PBVv6wsZZnN9Wy7/Bx4gLGvAk5XDOtkEUVQfIzNE9BRjYVgcgQcc6xpbaVZzeGRgq7m45hBnNKc7h2eiGXT85nXG6qFsKTEUdFIBIBzjl21LeHRgoba3mroR2AMZnJzJ+YxyUTc5k/MZcxWSkeJxVREYgMi92N7by86zBrdjWxZtdhjhzvAmB8XhrzJuT2FUOelrsQD6gIRIZZb69je30bq8PF8OruZtpOdANQVpDB/ImhYpg7IVeXp8qwUBGIeKy7p5dNh1pZHR4tvLa3mc6uXgIG08Zmhoshj/eXZpOaqBnOMvRUBCIjzInuHtbvbwmNGHYf5o39R+jqccQHjNklWcyfkMv8iXnMLsnSInkyJFQEIiPc8ZPdrNt3hNW7DrN612E21rTQ6yApPkBlaTaXTMxj/sRcZozNJD4u4HVciUKerT5qZlXAvwFxwAPOuXvetb0E+AWQFd7nG865P0Yyk8hIlJoYzwcn5fPBSfkAtHZ2sXZ3c7gYmvj+iu0ApCfFM7ski1nFWcwsymJGcSajM5K9jC4xIGIjAjOLA3YAHwJqgNeAxc65Lf32WQq84Zy738ymAn90zpWe7Xk1IhA/Otx+glf3NPPyziZe39/Cjvo2enpD/3fHZCYzsziLGUVZzCzOZPrYTC2aJ+/h1YhgDrDTObc7HOJXwA3Aln77OGBU+PNM4FAE84hErdz0JK6dXsi10wsB6DjZw+ZDR1l/oIU3a472LYcBYAYT89OZWZTFrOJMZhRlUV6YQVK8zjXI6UWyCMYCB/p9XQPMfdc+3wJWmtmXgDTgqtM9kZndDtwOUFJSMuRBRaJNSmIclaU5VJbm9D125NhJNtSEi+FACy/taOC3r9cAkBgXYMqYUcwsymRmURYzi7OYkJdGQPd0Fry/Q9liYJlz7odmNh94xMymOed6++/knFsKLIXQoSEPcoqMeNlpiSwoG82CstFAaObzoaOdbDjQEvqoaeG362p4eM0+ADKS4plelMnM4qxQQRRnERyVrOUxfCiSRXAQKO73dVH4sf5uA6oAnHNrzCwZyAMaIphLxBfMjLFZKYzNSuk7pNTT69jV2N5XDBsOHOWBP++mqyf0+9XojKTQuYaiTMoLR1EezKAoO0XlEOMiWQSvAZPMbDyhAvhr4OZ37bMfuBJYZmZTgGSgMYKZRHwtLmBMLshgckEGn6gM/Z7W2dXD1tpWNoTPN6yvaeH5rfV9fyc9KZ6yYAZlwQzKgxmUB0dRFszQjOgYErEicM51m9kXgRWELg19yDm32cz+L1DtnPsd8FXg52b2FUInjv/GRdvEBpEol5wQx+ySbGaXZPc91n6im+11bWyva2NbXSvb6tr4/YZD/PLV7r59xmQmh8ohPHIoD45iQn4aCZrnEHU0oUxEzotzjrrWTrbVtbGtto3t4YLY1djed2gpIc6YmJ8eKobCUX2jCJ178J5nE8pEJHaYGYWZKRRmprAwfEIa4GR3L7ub2tle18bWcEGs3dPM0+vfvho8MyWBsmAGU4IZlAVHUV6YQVlBBmlJ+hE0EuhfQUQGJTE+QHlwFOXBUdww6+3Hjx7vYnv924eWttW28sS6Go6d7OnbZ0xmMuPz05iQl86E/DTG56UxMT+dMVkpxOnS1mGjIhCRiMhMTWDO+BzmjH97rkNvr+NgS0dfMexuOsbupmM8vf4gbZ1vn39IjA9QmpvKhLz0cFGkMSE/nQl5aWTrXtFDTkUgIsMmEDCKc1IpzknlQ1ML+h53ztHUfpI9TcfY3dgeKojGY+xoaOP5rfV09759LjM7NYEJ+emMz0tjQr+SGJebqtnTA6QiEBHPmRn5GUnkZyS9YwQB0NXTS82RjlBBNB4Ll0Q7L+1o5Il1NX37BQzGZqf0HWY6VRCleWkERyXrUNNZqAhEZERLiAswPi90/uDKKe/c1tbZFR5FvF0QuxuPsXZPMx1dPf2ewyjKTqUoO4WSnNS+j1OjE7/PiVARiEjUykhOYEZRaOXV/k5d6rq78Rj7Dh9nf/NxDjSH/tx4sJaW8P2kT8lMSXhXObxdGGOyUmJ+boSKQERiTv9LXS+96L3bj3Z0cSBcDgeOhApif3MHW2pbWbmlrm9eBIQOOY3JSqE4OzySyA2VRUlOKsXZKeSkJUb9HAkVgYj4TmZKApljM5k2NvM923p6HfWtneFyON5XGPubj/OnbQ00tZ94x/5piXEU56QyNiuFMeGPsdkpjM1KZkxWCqMzRv75CRWBiEg/cQHr+4E+b0Lue7YfP9nNgeaOvnLY33ycmiPHOdjSSfW+Ixzt6HrP8wVHJYeLIrmvKMaEFwQck5VCuscT61QEIiIXIDXx7UX4Tqf9RDe1LR3UtHRwqO+jk4MtHVTvO0Ldm7XvuBwWYFRy/DuKYbhHFSoCEZEhlJ4Uz6SCDCYVnL4oenodDW2dHGrp4GBLZ7+y6DjnqOKzl5byuQ9OGPLMKgIRkWEUF3j7RPbF406/T1tnF7VHQ6OI/qOK/IykiGRSEYiIjDAZyQlkJCcw+QyjiqEW2xfHiojIOakIRER8TkUgIuJzKgIREZ9TEYiI+JyKQETE51QEIiI+pyIQEfE5c86de68RxMwagX0D/Ot5QNMQxvGS3svIFCvvJVbeB+i9nDLOOZd/ug1RVwSDYWbVzrlKr3MMBb2XkSlW3kusvA/QezkfOjQkIuJzKgIREZ/zWxEs9TrAENJ7GZli5b3EyvsAvZdz8tU5AhEReS+/jQhERORdVAQiIj7nmyIwsyoz225mO83sG17nGSgzKzazF8xsi5ltNrO7vM40GGYWZ2ZvmNnvvc4yGGaWZWZPmNk2M9tqZvO9zjRQZvaV8PfWJjN7zMySvc50vszsITNrMLNN/R7LMbPnzOyt8J/ZXmY8X2d4L98Pf4+9aWZPmVnWULyWL4rAzOKA/wCuAaYCi81sqrepBqwb+KpzbiowD7gzit8LwF3AVq9DDIF/A5Y758qBmUTpezKzscCXgUrn3DQgDvhrb1NdkGVA1bse+wbwJ+fcJOBP4a+jwTLe+16eA6Y552YAO4B/HIoX8kURAHOAnc653c65k8CvgBs8zjQgzrla59zr4c/bCP3AGettqoExsyLgOuABr7MMhpllApcBDwI4504651q8TTUo8UCKmcUDqcAhj/OcN+fcKqD5XQ/fAPwi/PkvgI8Oa6gBOt17cc6tdM51h798BSgaitfySxGMBQ70+7qGKP3h2Z+ZlQKzgVe9TTJgPwb+Aej1OsggjQcagf8KH+Z6wMzSvA41EM65g8APgP1ALXDUObfS21SDVuCcqw1/XgcUeBlmCP0t8OxQPJFfiiDmmFk68Fvg75xzrV7nuVBmdj3Q4Jxb53WWIRAPvA+43zk3GzhG9Bx+eIfw8fMbCJXbGCDNzD7tbaqh40LXy0f9NfNm9k1Ch4kfHYrn80sRHASK+31dFH4sKplZAqESeNQ596TXeQboUuAjZraX0KG6K8zsv72NNGA1QI1z7tTI7AlCxRCNrgL2OOcanXNdwJPAJR5nGqx6MysECP/Z4HGeQTGzvwGuBz7lhmgimF+K4DVgkpmNN7NEQie/fudxpgExMyN0LHqrc+5er/MMlHPuH51zRc65UkL/Hv/rnIvK3zydc3XAATMrCz90JbDFw0iDsR+YZ2ap4e+1KyB+3/gAAAC8SURBVInSE9/9/A74TPjzzwDPeJhlUMysitDh1I84544P1fP6ogjCJ1e+CKwg9E39uHNus7epBuxS4BZCv0GvD39c63Uo4UvAo2b2JjAL+I7HeQYkPKp5Angd2EjoZ0TULNFgZo8Ba4AyM6sxs9uAe4APmdlbhEY893iZ8Xyd4b38FMgAngv/3//ZkLyWlpgQEfE3X4wIRETkzFQEIiI+pyIQEfE5FYGIiM+pCEREfE5FICLicyoCERGf+//8HozHldzVCAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "gbiznzHfntmM",
        "outputId": "6e37ceaf-8bef-44ba-f7de-576e8227cacc"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(val_accs)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD4CAYAAAD2FnFTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dcnCSQsYQ9r2FdZBRFwqVrRglZFa7W4VG3dbm+1y7V16a/Xa229XexiF2ur1irWSrlKKy4tUJe6gEpIWCQIhC0LEAIkISF75vP7I4ONmIUlk5PJvJ+PRx5mzpwz8wbJvHPO95zzNXdHRETkSHFBBxARkbZJBSEiIg1SQYiISINUECIi0iAVhIiINCgh6AAtpU+fPj5s2LCgY4iIRJXVq1fvc/eUhp5rNwUxbNgw0tLSgo4hIhJVzGxnY8/pEJOIiDRIBSEiIg1SQYiISINUECIi0iAVhIiINEgFISIiDVJBiIhIg1QQIiL1VFTXsmDlDvKKyoOOEjgVhIhIWG3I+cbCNdz7wgZm/+wNHvrnZsqraoOOFRgVhIgI4O58/6VM/rFhD187dxTnndSPh/65hdk/e4Mla3cRi5OrqSBERIA/vL2dJ1fs4MYzh/NfnxnLb66exqJbT6Nnl4587dkMrvjdStbnFgcds1WpIEQk5r24dhc/eHkjn500gP934UkfLZ8xvBdLbjuTH31uEtv3HeKSh9/mrufWUVBSGWDa1qOCEJGY9t62/dyxaC2nDuvJz66cQlycfez5+Dhj/owhvP7tc7j5UyNYnJHLp3/6Bo++uZWqmlBAqVuHCkJEYtaW/BJuXpDG4F6deOy66SR1iG903W5JHfjOhSex9BtnMXN4L/73lQ+Z89CbvLoxv92OT6ggRCQm5R+s4IY/riKxQzxPfmkGPTp3PKrtRqR05Q83nMqTXzqVOIMbn0rj+j+uImtvSYQTtz4VhIjEnJKKam744yqKyqr44w2nMrhX52N+jXPG9uUf3ziLey8aT0Z2IXMeeovvvbiB4rLqCCQORkQLwszmmtkmM8sys7sbeH6Imb1uZhlmts7MLmzg+VIz+1Ykc4pI7KiqCfGVP6WzOb+E3157ChMHdT/u1+oQH8eXzxzOG986h/mnDuapFTs456ev8/S7O6mpjf7xiYgVhJnFAw8DFwDjgavMbPwRq30XWOTuU4H5wG+PeP7nwN8jlVEkmmXvLyMUap/HviPF3bl78TreztrHDz83ibPHNDjT5jHr3TWRBy6bxEu3f4qx/ZP57799wEW/fpsVW/e1yOsHJZJ7EDOALHff5u5VwEJg3hHrONAt/H13YNfhJ8zsUmA7sCGCGUWiTm3IeeDlTM568HW+uWgN1e3gN9XW8vPlm1mcnsc3zxvDldMHt/jrjx/YjWdvnsUj10yjtLKGqx97j/94ejU5B8pa/L1aQyQLYhCQU+9xbnhZffcB15pZLvAKcDuAmXUF7gK+F8F8IlGntLKGWxak8dhb25k1ohcvrNnFfz6TTkV17N4O4mj9+b1sfv1aFl+YPpivzR4VsfcxMy6YNIB//tfZfOszY3hzSwGzf/4vHlz6IYcqayL2vpEQ9CD1VcCT7p4KXAg8bWZx1BXHL9y9tKmNzewWM0szs7SCgoLIpxUJUG5hGZ9/ZAVvbC7g+/MmsPCW0/j+vAksz8znpqfSKKuKrg+f1vTqxny++7f1nDM2hR9cNhEza36jE5TUIZ7bzh3Na3ecw0WTBvDw61s5+8E3+MXyzeQfrIj4+7cEi9T5u2Z2GnCfu88JP74HwN1/WG+dDcBcd88JP94GzAKeBw7v//UAQsC97v6bxt5v+vTpnpaWFok/ikjg0rMLuWVBGpU1IR6+ehpn1Tt2/vzqXL793FqmDunJEzecSvdOHQJM2vaszSli/qPvMrJvF/5yy2l0SUwIJEd6diG/fnULb2wuIN6MORP7c92socwY3qtVCqsxZrba3ac3+FwECyIB2AzMBvKAVcDV7r6h3jp/B/7i7k+a2UnAq8AgrxfKzO4DSt39p029nwpC2qsX1uTx7efWMaB7En+4fjqj+iZ/Yp1/fLCb25/NYHTfZBbcOIM+XRMDSNr27Nx/iM/9dgWdOsaz+D9Pp29yUtCR2Ln/EH96dyeL0nIpLq9mXP9kvnjaUC49eVAg5RVIQYTf+ELgISAeeMLdHzCz+4E0d18SPqvpMaArdQPWd7r7siNe4z5UEBKDQiHnoVe38KtXtzBjeC9+f+0p9OzS+MVc/9pcwK1PpzGwRyeeuWkmA7p3asW0bc+BQ1Vc/sgKCsuqeP4rpzMypWvQkT6mvKqWJWvzeGrFTjJ3HyQ5KYHPn5LKF2cNZUQrZg2sIFqTCkLak4rqWu74v7W8vG43V5ySygOXTaJjQvNDhu9vP8CNT66ie+cOPHPTTIb27tIKadue8qparn78XTJ3HeTPN8/klKG9go7UKHcnPbuQp1bs5O8f7Ka61jlrTArXzRrKp8f1JT4usoefVBAiUWTvwQpuXpDGurxi7p47jlvOGnFMx6jX5xZz3RPv0SE+jj/dNJMx/T55SKo9qw05X/nTapZvzOeRa6Yxd+KAoCMdtb0lFSx8P4c/v5fNnoMVpPbsxLWzhvKF6YOb3Hs8ESoIkSjxQV4xNy9Io7i8ml/On8r54/sd1+tsyS/hmsffo7o2xIIvz2RS6vFfLRxN3J3/WbKBBSt3ct/F47nhjOFBRzou1bUhlmfms2DlDt7ddoDEhDgunjKQ608b1uL/L1UQIlFg6YY9fGPhGnp27sBj109nwsAT+yDI3l/G1Y+/S1FZNU/ccCozhrfdwywt5Xf/2sqP/v4ht5w1gu/Um9chmm3aU8KClTv4a0YeZVW1nDy4B9efPpQLJw0gMaHxu88eLRWESBvm7vz+zW38+B8fMjm1B49dd0qLnW2zu7icax9/j7yicn537SmcM7Zvi7zusXB3Vm7bT+6Bckb27cqovl0jciruC2vy+PrCNVw0eQC/mj/1E/M6RLuDFdU8vzqXp1fuZNu+Q/Tu0pH5MwZzzcyhDOxx/CckqCBE2qiqmhDf+et6nludy0WTB/DTK6Y0OSfB8dhXWsl1f3ifLXtL+NX8qVwwqXWOyVfVhHhx7S4ef3s7G3cf/Nhz/bolMrpvMqPChTG6b1dG90um13EeZ1+xdR/XP/E+04b0ZMGNM1rkN+u2KhRy3tm6jwUrd/LqxnwALp4ykIe+cPJxXU+hghBpgw4cquI/nl7N+zsO8PXZo/nGeaMjdsFUcXk1X35yFRnZhTz4+SlcfkpqRN4HoLismmfe38lTK3aQf7CSMf26ctOZIzh1eC+2FZSyZW8pW/JLydpbQtbeUg5V/fs2Ib27dPxEaYzu25WU5MRG/24+3HOQKx5ZSf/uSTz3H6fTvXPsXCiYW1jGM+9lEwo59xznITUVhEgbsyW/hBufSiP/YAUPXjGFS6YMjPh7llXVcPOCNN7J2s/98yZw3WnDWvT1s/eX8cQ721mUlkNZVS1njurDTZ8aztljUhr9cHd3dhdXhEujrjAOf3+w4t+3DumWlBAujWRG9wsXSL9kDLj8kRXUhpy/fvUMBp3AoZZYpYIQaUP+tbmA255JJ7FDPI9ddwpTh/RstfeuqK7l9mczWJ6Zz7fnjOWrnz7xm9at3lnI429tY+mGPcTHGRdPGchNZ45g/MBuzW/cCHenoLSSrPxwYewtYUt+KVsLStlXWvXRembQpWMCf7l11gkP6seqpgoimJuSiMSop1bs4P6XMhnTL5nHr5/e6r/xJnWI57fXTONb/7eWB5duorSyhjvnjD3mQ1u1IWfZhj089tY20rOL6JaUwK1nj+T604bRv/uJD7CbGX2Tk+ibnMTpo/p87LkDh6rCexol7NxfxgUT+6scIkQFIRJWXRviYHk1XRITSEyIa9HxgJraEN97MZOn393JeSf145fzTw7spnEd4uP4xZV17//IG1s5VFnDfRdPOKqzfg5V1vB/aTk88c4Osg+UMbhXJ+67eDxXTB/can+eXl06MmN4r5g4bTdoKgiJaWVVNby5uYBlG/J59cO9FJfXzSecEGd0SUyg6+GvpAS6JCaQnJhAl8R4uiZ2oGti/EfLu9b76pKYQHK95ZU1IW77czpvbdnHrWeN4M654yJ++4TmxMUZD1w6ka6JCTz65jZKK2v4yeWTSYhv+HYe+QcreHLFDv78XjbF5dVMG9KDey4Yx2cm9A/8zyKRo4KQmFN4qIp/bsxnWWY+b24uoLImRPdOHZh9Ul8mDepOWVUtpZU1HKqsobSyhtKKGg5V1VBcVkVeYVn4uVoOVdVwtEN4HeKNn1w+mStPbflZzI6XmXHPBeNITkzgZ8s3U1ZZyy+vOvljp4hu3H2Qx9/azpK1edSGnDkT+nPTp0ZwytDWGzeR4KggJCbkFZWzbMMelm3I5/0dB6gNOQO6JzH/1MHMmdCfU4f3okMjvz03JhRyyqprOVRZQ0lFXaEcqqyhpH65VNZQVlnLuSf1ZVorDkYfLTPj9tmj6ZKYwP0vZXLzgtX87tpprNpRN/D81pZ9dO4YzzUzh/KlM4bF7M3/YpXOYpJ2yd3ZsreUZRv2sHRDPuvzigEY1bcrcyb0Y86E/kwa1D3QiVramkWrcrhr8Tq6dEygtLKGvsmJ3HDGMK6ZMTSmri2INTqLSWJCKORk5BSxLLNuT2H7vkMATB3Sg7vmjuMzE/q1uTkB2pIrTx1M16QEnl65k8+fksrFUwYe1S3Gpf1SQUhUq6oJsXLbfpZt2MPyzHz2llSSEGecNrI3Xz5zOJ8Z349+3YKfRSxaXDhpABe20q04pO1TQUjUKS6v5p2sfSzdsIfXPtxLSUUNnTrEc87YFOZM6M+nx/XVvMwiLUAFIW1eRXUtq3cW8k7WPt7Zup/1uUWEHHp27sDcCf2ZM6E/Z47u0+I3uROJdSoIaXNqQ876vOK6QsjaR9rOQqpqQsTHGScP7sFt547mzFF9mDakR6Pn7YvIiVNBSODcna0Fpby9pW4P4d1t+ykJ36htXP9kvjhrKGeM6s2M4b3pGtDVxyKxSD9tEohdReW8k7WPFVv3807WPvaWVAIwuFcnPjtpAKeP6sPpI3vTp2tiwElFYpcKQlpFUVkVK7fu552t+1iRtZ9t4VNQe3fpyGkje3PmqD6cMaoPg3t1DjipiBymgpCI2bj7IEvW7uLtLfv4YFcx7tClYzwzhvfi6plDOGNUH8b2S253U0OKtBcqCGlRe0sqWLJmF8+n57Fx90ES4oxpQ3ryjdljOGNUb6YM7nHMt7QQkWCoIOSEVVTXsjwzn8Xpuby5ZR+1IWdKane+d8kELp4y8LjnGRaRYKkg5Li4O6t2FLI4PZeX1+2mpLKGAd2TuPWsEXxu2iBG9U0OOqKInCAVhByTnfsPsTg9j8UZueQcKKdzx3jmTuzP5dNSmTWit+YGEGlHVBDSrOLyal5et5vF6bmk7SzEDM4Y2YdvnjeGORP6BzYzmohEln6ypUHVtSHe2lLA8+l5LM/Mp6omxKi+Xblr7jgunTqQAd1bdy5lEWl9Kgj5iLuzYddBFqfnsWRtHvtKq+jVpSNXzxjC5dNSmTiom+ZPEIkhKggB4G8ZeTzyxlY25ZfQMT6O2Sf15XPTUjl7TIrmBBCJUSoIISO7kP9atIZx/bvxg0snctHkAfTorFNTRWKdCiLGVdWEuPv59fTrlsRfbp1FcpLmURCROiqIGHf4sNIfrp+uchCRj4nowWUzm2tmm8wsy8zubuD5IWb2upllmNk6M7swvPx8M1ttZuvD/z03kjlj1eb8En7z+hYumTKQ2Sf1CzqOiLQxEduDMLN44GHgfCAXWGVmS9w9s95q3wUWufsjZjYeeAUYBuwDLnb3XWY2EVgKDIpU1lhUG3Luen4dXRMT+J+LxwcdR0TaoEjuQcwAstx9m7tXAQuBeUes40C38PfdgV0A7p7h7rvCyzcAncxMEwO0oKdW7CAju4j/uXgCvTXngog0IJJjEIOAnHqPc4GZR6xzH7DMzG4HugDnNfA6lwPp7l4ZiZCxKOdAGQ8u3cSnx6Yw7+SBQccRkTYq6BPcrwKedPdU4ELgaTP7KJOZTQB+DNza0MZmdouZpZlZWkFBQasEjnbuznf+up44gx9cNkkXvolIoyJZEHnA4HqPU8PL6rsRWATg7iuBJKAPgJmlAn8FrnP3rQ29gbs/6u7T3X16SkpKC8dvn55Pz+OtLfu4+4JxDOqh22WISOMiWRCrgNFmNtzMOgLzgSVHrJMNzAYws5OoK4gCM+sBvAzc7e7vRDBjTCkoqeT7L2Vy6rCeXDNzaNBxRKSNi1hBuHsNcBt1ZyBtpO5spQ1mdr+ZXRJe7Q7gZjNbCzwL3ODuHt5uFHCvma0Jf/WNVNZYcd+SDZRX1/Kjyydrmk8RaVZEL5Rz91eoO3W1/rJ7632fCZzRwHY/AH4QyWyx5h8f7OHl9bv59pyxjEzpGnQcEYkCQQ9SSysoLq/m3hc+4KQB3bjlrBFBxxGRKKGCiAE/fGUj+0or+cnlk+kQr//lInJ09GnRzq3I2sfCVTncfNYIJqV2DzqOiEQRFUQ7Vl5Vy92L1zOsd2e+ed6YoOOISJTR3VzbsZ8v30T2gTKevXkWSR3ig44jIlFGexDt1NqcIv7w9naunjmE00b2DjqOiEQhFUQ7VFUT4q7n15GSnMjdF4wLOo6IRCkdYmqHfv+vrXy4p4THr5tON00CJCLHSXsQ7UzW3hJ+/VoWF00ewHnjNQmQiBw/FUQ7Uhty7nxuHZ0T47nvkglBxxGRKKeCaEeeXrmD9Owi7r1oPH00CZCInCAVRDuRW1jGT5Zu4uwxKVw2VbOzisiJU0G0A3WTAH0AwAOXTdQkQCLSIlQQ7cBfM/J4c3MBd80dR2rPzkHHEZF2QgUR5QpKKrn/pUymD+3JF2dpEiARaTkqiCh334sbKKvUJEAi0vJUEFFs2YY9vLxuN1+bPYpRfTUJkIi0LBVElDpYUc1/v/AB4/onc+vZI4OOIyLtkG61EaV++MqHFJRU8th10zUJkIhEhD5ZotDKrft59v1sbvrUCCan9gg6joi0UyqIKFNeVcs9i9cxVJMAiUiE6RBTFKmoruXO59exY38Zf755Jp06ahIgEYkcFUSUyC0s4yt/Smd9XjHfnjOW00f2CTqSiLRzKogosCJrH7c9m0F1TYjHrpvO+bqNt4i0AhVEG+buPP7Wdn74942MSOnK7794CiNTdL2DiLQOFUQbVVZVw13Pr+fFtbuYO6E/P71yCl0T9b9LRFqPPnHaoJ37D3Hr06vZlF/CnXPH8pWzR+oOrSLS6lQQbcwbm/bytWczMDOe+tIMzhqTEnQkEYlRKog2IhRyfvtGFj9bvplx/bvx+2tPYUhv3bpbRIKjgmgDSiqquWPRWpZl5jPv5IH86HOTdY2DiAROBRGwrL2l3Pp0Gjv2l/HfF43ny2cM03iDiLQJKogALd2whzsWrSUxIY4/3TiT00b2DjqSiMhHVBABqA05D/1zM79+LYspqd155NpTGNijU9CxREQ+RgXRyorLqvn6XzJ4Y1MBV05P5f55E0nqoPEGEWl7VBCt6MM9B7n16dXsKirnB5dO5JqZQzTeICJtVkRv921mc81sk5llmdndDTw/xMxeN7MMM1tnZhfWe+6e8HabzGxOJHO2hhfX7uKyh1dQXlXLwltO49pZQ1UOItKmNbkHYWbdgXuAS4G+gAN7gReAH7l7URPbxgMPA+cDucAqM1vi7pn1VvsusMjdHzGz8cArwLDw9/OBCcBA4J9mNsbda4/zzxmYmtoQP1m6iUff3Mb0oT357TXT6NstKehYIiLNam4PYhFQCJzj7r3cvTfw6fCyRc1sOwPIcvdt7l4FLATmHbGOA93C33cHdoW/nwcsdPdKd98OZIVfL6ocOFTF9X98n0ff3MZ1pw3lzzfPUjmISNRobgximLv/uP4Cd98D/NjMvtzMtoOAnHqPc4GZR6xzH7DMzG4HugDn1dv23SO2HXTkG5jZLcAtAEOGDGkmTuvaVVTOFb9bSUFpJQ9+fjJXTB8cdCQRkWPS3B7ETjO708w+moDAzPqZ2V18/MP/eF0FPOnuqcCFwNNmdtTjIu7+qLtPd/fpKSlt655FL63bRV5ROX+5ZZbKQUSiUnMfxl8AegP/MrNCMzsAvAH0Aq5sZts8oP4nY2p4WX03Ej5U5e4rgSSgz1Fu26al7yxiaO/OTB3SM+goIiLHpcmCcPdC4I/AbcDg8DjESe5+F82PCawCRpvZcDPrSN2g85Ij1skGZgOY2UnUFURBeL35ZpZoZsOB0cD7x/ZHC467k55dyNTBPYKOIiJy3JosCDP7GnVnLN0GfGBm9QeZ/7epbd29JrzdUmAjdWcrbTCz+83skvBqdwA3m9la4FngBq+zgbo9i0zgH8BXo+kMpt3FFewtqdTeg4hEteYGqW8GTnH3UjMbBjxnZsPc/ZdAsyfxu/sr1J26Wn/ZvfW+zwTOaGTbB4AHmnuPtig9uxCAaSoIEYlizRVEnLuXArj7DjM7h7qSGMpRFESsysguIjEhjnEDkoOOIiJy3JobpM43s5MPPwiXxUXUDSRPimSwaJaRXcjk1O50iI/oheoiIhHV3CfYdcCe+gvcvcbdrwPOiliqKFZZU8sHeQd1eElEol6Th5jcPbeJ595p+TjRL3PXQapqQ0wdojOYRCS66RhIC8vIrrs9lc5gEpFop4JoYenZhQzsnkQ/3XNJRKKcCqKFZWQXMXWo9h5EJPqpIFrQ3oMV5BWV6wpqEWkXVBAtKCNH4w8i0n6oIFpQenYhHePjmDioW/Mri4i0cSqIFpSRXcT4gd1ITIgPOoqIyAlTQbSQmtoQ63KLdP2DiLQbKogW8uGeEiqqQ7qCWkTaDRVEC8kI38FVexAi0l6oIFpIRnYRKcmJDOrRKegoIiItQgXRQtKzC5k2pAdmugu6iLQPKogWcOBQFTv2l+n6BxFpV1QQLWBNTnj8QVdQi0g7ooJoAek7i4iPMyaldg86iohIi1FBtICMnEJOGpBM547NzeAqIhI9VBAnqDbkrM0pZupgjT+ISPuigjhBW/aWUFpZo+sfRKTdUUGcoMMzyOkKahFpb1QQJygju5CenTswtHfnoKOIiLQoFcQJysguYuqQnrpATkTaHRXECSgur2bL3lKmafxBRNohFcQJWKsZ5ESkHVNBnICM7CLMYLIukBORdkgFcQLSswsZ2y+Z5KQOQUcREWlxKojjFAo5a3I0g5yItF8qiOO0ff8hisurdQW1iLRbKojjlL5TM8iJSPumgjhOGTlFJCclMDKla9BRREQiQgVxnDKyizh5cA/i4nSBnIi0TxEtCDOba2abzCzLzO5u4PlfmNma8NdmMyuq99xPzGyDmW00s19ZG7pUubSyhk17Dur6BxFp1yI2gYGZxQMPA+cDucAqM1vi7pmH13H3b9Zb/3Zgavj704EzgMnhp98GzgbeiFTeY7Eut4iQoyuoRaRdi+QexAwgy923uXsVsBCY18T6VwHPhr93IAnoCCQCHYD8CGY9Jofv4HqyphgVkXYskgUxCMip9zg3vOwTzGwoMBx4DcDdVwKvA7vDX0vdfWMD291iZmlmllZQUNDC8RuXkV3IiJQu9OjcsdXeU0SktbWVQer5wHPuXgtgZqOAk4BU6krlXDP71JEbufuj7j7d3aenpKS0SlB3JyO7SPM/iEi7F8mCyAMG13ucGl7WkPn8+/ASwGXAu+5e6u6lwN+B0yKS8hjlHChn/6EqXf8gIu1eJAtiFTDazIabWUfqSmDJkSuZ2TigJ7Cy3uJs4GwzSzCzDtQNUH/iEFMQMnLCF8jpCmoRaeciVhDuXgPcBiyl7sN9kbtvMLP7zeySeqvOBxa6u9db9hywFVgPrAXWuvuLkcp6LNJ3FtK5Yzxj+ycHHUVEJKIidporgLu/ArxyxLJ7j3h8XwPb1QK3RjLb8crIKWJKag/idYGciLRzbWWQOipUVNeSueugxh9EJCaoII7B+rxiakKuM5hEJCaoII5BRnbdAPXJ2oMQkRiggjgGGdlFDOnVmT5dE4OOIiIScSqIo+TupGcXavxBRGKGCuIo7S6uIP9gpcYfRCRmqCCO0uEb9GkPQkRihQriKKVnF5KYEMe4/t2CjiIi0ipUEEcpI7uQyand6ZigvzIRiQ36tDsKlTW1fLBLM8iJSGxRQRyFzF0HqaoJMVUTBIlIDFFBHIXDA9TThmoPQkRihwriKGTkFDGwexL9uiUFHUVEpNWoII5CRnahxh9EJOaoIJqxt6SC3MJyXf8gIjFHBdGMf18gpz0IEYktKohmZGQX0SHemDBQF8iJSGxRQTQjPbuQ8QO7k9QhPugoIiKtSgXRhJraEOtyi5im8QcRiUEqiCZ8uKeEiuqQxh9EJCapIJpweAY5XUEtIrFIBdGEjOwiUpITSe3ZKegoIiKtTgXRhIycIqYO7oGZBR1FRKTVqSAaceBQFdv3HdL4g4jELBVEI9bk1I0/6AwmEYlVKohGZGQXER9nTErtHnQUEZFAqCAakZ5dyLj+yXTumBB0FBGRQKggGlAbctbmFDNN4w8iEsNUEA3I2ltKaWWN7uAqIjFNBdGAjy6Q0x6EiMQwFUQD0rML6dm5A8N6dw46iohIYFQQDcjILmLqkJ66QE5EYpoK4gjF5dVs2Vuq+y+JSMxTQRxhbY5mkBMRgQgXhJnNNbNNZpZlZnc38PwvzGxN+GuzmRXVe26ImS0zs41mlmlmwyKZ9bCM7CLMYMpgXSAnIrEtYleBmVk88DBwPpALrDKzJe6eeXgdd/9mvfVvB6bWe4kFwAPuvtzMugKhSGWtLyOnkDF9k0lO6tAabyci0mZFcg9iBpDl7tvcvQpYCMxrYv2rgGcBzGw8kODuywHcvdTdyyKYFYBQyMMD1Bp/EBGJZEEMAnLqPc4NL/sEMxsKDAdeCy8aAxSZ2WIzyzCzB8N7JEdud4uZpZlZWkFBwQkH3r7/EMXl1bqCWkSEtjNIPR94zt1rw48TgE8B3wJOBUYANxy5kbs/6u7T3X16SkrKCYfIyKi2NcUAAAXqSURBVD48QK09CBGRSBZEHjC43uPU8LKGzCd8eCksF1gTPjxVA/wNmBaRlPWkZxeSnJTAyJSukX4rEZE2L5IFsQoYbWbDzawjdSWw5MiVzGwc0BNYecS2Pczs8G7BuUDmkdu2tIzsIk4e3IO4OF0gJyISsYII/+Z/G7AU2AgscvcNZna/mV1Sb9X5wEJ393rb1lJ3eOlVM1sPGPBYpLICHKqsYdOeg7r+QUQkLKKTHbj7K8ArRyy794jH9zWy7XJgcsTCHWFtbhEh1/iDiMhhbWWQOnAfDVDrFhsiIoAK4iMZ2UWMSOlCj84dg44iItImqCAAd2dNTiFTB2v8QUTkMBUEkHOgnH2lVRp/EBGpRwVB3f2XAF1BLSJSjwqCuvGHzh3jGdNPF8iJiBymgqDuCurJqd1JiNdfh4jIYTH/iVhRXUvmroM6vCQicoSYL4iDFdV8dvIAzhzVJ+goIiJtSkSvpI4GfZOT+OX8qc2vKCISY2J+D0JERBqmghARkQapIEREpEEqCBERaZAKQkREGqSCEBGRBqkgRESkQSoIERFpkNWbCjqqmVkBsPMEXqIPsK+F4kRaNGWF6MobTVkhuvJGU1aIrrwnknWou6c09ES7KYgTZWZp7j496BxHI5qyQnTljaasEF15oykrRFfeSGXVISYREWmQCkJERBqkgvi3R4MOcAyiKStEV95oygrRlTeaskJ05Y1IVo1BiIhIg7QHISIiDVJBiIhIg2K+IMxsrpltMrMsM7s76DxNMbPBZva6mWWa2QYz+3rQmZpjZvFmlmFmLwWdpTlm1sPMnjOzD81so5mdFnSmxpjZN8P/Bj4ws2fNLCnoTPWZ2RNmttfMPqi3rJeZLTezLeH/tol5fhvJ+mD438E6M/urmfUIMmN9DeWt99wdZuZm1iJTZMZ0QZhZPPAwcAEwHrjKzMYHm6pJNcAd7j4emAV8tY3nBfg6sDHoEEfpl8A/3H0cMIU2mtvMBgFfA6a7+0QgHpgfbKpPeBKYe8Syu4FX3X008Gr4cVvwJJ/MuhyY6O6Tgc3APa0dqglP8sm8mNlg4DNAdku9UUwXBDADyHL3be5eBSwE5gWcqVHuvtvd08Pfl1D3ATYo2FSNM7NU4LPA40FnaY6ZdQfOAv4A4O5V7l4UbKomJQCdzCwB6AzsCjjPx7j7m8CBIxbPA54Kf/8UcGmrhmpEQ1ndfZm714QfvguktnqwRjTydwvwC+BOoMXOPIr1ghgE5NR7nEsb/sCtz8yGAVOB94JN0qSHqPsHGwo6yFEYDhQAfwwfEnvczLoEHaoh7p4H/JS63xR3A8XuvizYVEeln7vvDn+/B+gXZJhj8GXg70GHaIqZzQPy3H1tS75urBdEVDKzrsDzwDfc/WDQeRpiZhcBe919ddBZjlICMA14xN2nAodoO4dAPiZ87H4edaU2EOhiZtcGm+rYeN359W3+HHsz+3/UHdp9JugsjTGzzsB3gHtb+rVjvSDygMH1HqeGl7VZZtaBunJ4xt0XB52nCWcAl5jZDuoO3Z1rZn8KNlKTcoFcdz+8R/YcdYXRFp0HbHf3AnevBhYDpwec6Wjkm9kAgPB/9wacp0lmdgNwEXCNt+0LxkZS98vC2vDPWyqQbmb9T/SFY70gVgGjzWy4mXWkbqBvScCZGmVmRt0x8o3u/vOg8zTF3e9x91R3H0bd3+tr7t5mf8t19z1AjpmNDS+aDWQGGKkp2cAsM+sc/jcxmzY6oH6EJcD14e+vB14IMEuTzGwudYdHL3H3sqDzNMXd17t7X3cfFv55ywWmhf9Nn5CYLojwINRtwFLqfsAWufuGYFM16Qzgi9T9Nr4m/HVh0KHakduBZ8xsHXAy8L8B52lQeC/nOSAdWE/dz3Gbui2EmT0LrATGmlmumd0I/Ag438y2ULcX9KMgMx7WSNbfAMnA8vDP2e8CDVlPI3kj815te89JRESCEtN7ECIi0jgVhIiINEgFISIiDVJBiIhIg1QQIiLSIBWEiIg0SAUhIiIN+v+RIxiyOMFH4gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZ3Hr8WplthG"
      },
      "source": [
        "We then added option lemmas, MLP and weights to balance weights.\n",
        "The results are seen below "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jjSKO4orlYbQ",
        "outputId": "a93cded2-9ec1-438d-95c9-3fa9da2d6772"
      },
      "source": [
        "classifier_mlp_weights_lemmas = WSDClassifier(num_labels, device = 'cuda', use_mlp = True, hidden_size = 100,nbr_lemmas = len(lemma2i), lemma_embedding_size = 518, add_lemmas = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "lemmahidden 1286\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4s6pTO8_m02p"
      },
      "source": [
        "#for name, param in classifier.named_parameters():\n",
        "  #print(\"PARAM named %s, of shape %s\" % (name, str(param.shape)))\n",
        "  #print(param.requires_grad)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3nQUTyr5l4fj",
        "outputId": "e930755d-9dd1-4776-f3d7-3fdff97cdee6"
      },
      "source": [
        "# training\n",
        "from statistics import mean\n",
        "import numpy as np\n",
        "stop_early = 0\n",
        "early_stopping_patience = 6\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "LR = 0.0005\n",
        "n_epochs = 30\n",
        "import numpy as np\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "class_weight = compute_class_weight(\"balanced\", np.unique(list(label2i.values())),list(label2i.values()))\n",
        "\n",
        "class_weight = torch.tensor(class_weight, dtype=torch.float).to(classifier.device)\n",
        "\n",
        "#loss_function = nn.CrossEntropyLoss()\n",
        "loss_function = nn.NLLLoss(weight = class_weight) \n",
        "#optimizer = optim.SGD(classifier.parameters(), lr=LR)\n",
        "optimizer = optim.Adam(classifier_mlp_weights_lemmas.parameters(), lr=LR)\n",
        "\n",
        "config_name = 'sequoiaftb.asfalda_1_3.wsd.lr' + 'Adam' + str(LR) + '_bs' + str(BATCH_SIZE)\n",
        "out_model_file = './' + config_name + '.model'\n",
        "out_log_file = './' + config_name + '.log'\n",
        "\n",
        "\n",
        "# perte à chaque époque (sur le train / sur le validation set)\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "val_accs = []\n",
        "min_val_loss = None\n",
        "epoch_losses = []\n",
        "# pour tests: utiliser training sur dev\n",
        "#train_data = data['dev'] # data['train']\n",
        "train_data = wsd_data['train']\n",
        "val_data = wsd_data['val']\n",
        "dev_data = wsd_data['dev']\n",
        "\n",
        "val_accs = []\n",
        "\n",
        "print('Training.....')\n",
        "acc = 0\n",
        "for epoch in range(n_epochs):  # loop over the dataset multiple times\n",
        "  classifier_mlp_weights_lemmas.train()\n",
        "  \n",
        "  print('acc',acc)\n",
        "  print('Training..... epoch nr: ', epoch)\n",
        "  for b_tid_seqs, b_tg_trks, b_labels, b_lemma_idx in train_data.make_batches(64, shuffle_data=True):\n",
        "    \n",
        "  \n",
        "    optimizer.zero_grad()\n",
        "    b_tid_seqs = torch.tensor(b_tid_seqs, device=classifier_mlp_weights_lemmas.device).to(classifier_mlp_weights_lemmas.device)\n",
        "    b_tg_trks = torch.tensor(b_tg_trks, device=classifier_mlp_weights_lemmas.device).to(classifier_mlp_weights_lemmas.device)\n",
        "    b_labels = torch.tensor(b_labels, device=classifier_mlp_weights_lemmas.device).to(classifier_mlp_weights_lemmas.device)\n",
        "    b_lemma_idx = torch.tensor(b_lemma_idx, device=classifier_mlp_weights_lemmas.device).to(classifier_mlp_weights_lemmas.device)\n",
        "    log_probs = classifier_mlp_weights_lemmas(b_tid_seqs, b_tg_trks, b_lemma_idx).to(classifier_mlp_weights_lemmas.device)\n",
        "  \n",
        "    loss = loss_function(log_probs,  b_labels)\n",
        "    \n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_losses.append(loss.item())\n",
        "  epoch_losses.append(sum(train_losses)/len(train_losses))\n",
        " \n",
        "  pred_labels, val_losses, val_acc, _, _ = classifier_mlp_weights_lemmas.run_on_dataset(val_data, optimizer, batch_size=32, validation_use=True)\n",
        "  val_accs.append(val_acc)\n",
        "  print('--------')\n",
        "  print('train loss: ',epoch_losses[-1],'val accuracy: ', val_acc)\n",
        "  print('--------')\n",
        " \n",
        "  if val_acc>=acc:\n",
        "    acc=val_acc\n",
        "  else:\n",
        "    early_stopping_patience +=1\n",
        "    if early_stopping_patience == stop_early:\n",
        "      print('Stopping early...')\n",
        "      break\n",
        "\n",
        "  \n",
        "    \n",
        "  \n",
        " \n",
        "\n",
        "\n",
        "        \n",
        "  \n",
        "           \n",
        "# TODO HERE\n",
        "# training\n",
        "# - basic : train for NB_EPOCHS +\n",
        "# - BONUS : early stopping: stop epoch loop as soon as accuracy on dev decreases+\n",
        "\n",
        "# don't forget to toggle \n",
        "# - classifier.train() when training on train\n",
        "# - classifier.eval() when evaluating on val corpus\n",
        "\n",
        "\n",
        "print(\"train losses: %s\" % ' / '.join([ \"%.4f\" % x for x in epoch_losses]))\n",
        "print(\"val   losses: %s\" % ' / '.join([ \"%.4f\" % x for x in val_losses]))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training.....\n",
            "acc 0\n",
            "Training..... epoch nr:  0\n",
            "--------\n",
            "train loss:  2.427831932619044 val accuracy:  0.6674905838041432\n",
            "--------\n",
            "acc 0.6674905838041432\n",
            "Training..... epoch nr:  1\n",
            "--------\n",
            "train loss:  1.9151406714218198 val accuracy:  0.750882768361582\n",
            "--------\n",
            "acc 0.750882768361582\n",
            "Training..... epoch nr:  2\n",
            "--------\n",
            "train loss:  1.634779173158421 val accuracy:  0.7776012241054614\n",
            "--------\n",
            "acc 0.7776012241054614\n",
            "Training..... epoch nr:  3\n",
            "--------\n",
            "train loss:  1.4515471636342459 val accuracy:  0.7937853107344632\n",
            "--------\n",
            "acc 0.7937853107344632\n",
            "Training..... epoch nr:  4\n",
            "--------\n",
            "train loss:  1.3227205254959062 val accuracy:  0.8052024482109228\n",
            "--------\n",
            "acc 0.8052024482109228\n",
            "Training..... epoch nr:  5\n",
            "--------\n",
            "train loss:  1.225951257805105 val accuracy:  0.813382768361582\n",
            "--------\n",
            "acc 0.813382768361582\n",
            "Training..... epoch nr:  6\n",
            "--------\n",
            "train loss:  1.1484279026161517 val accuracy:  0.8211511299435028\n",
            "--------\n",
            "acc 0.8211511299435028\n",
            "Training..... epoch nr:  7\n",
            "--------\n",
            "train loss:  1.0861766667958674 val accuracy:  0.8237405838041432\n",
            "--------\n",
            "acc 0.8237405838041432\n",
            "Training..... epoch nr:  8\n",
            "--------\n",
            "train loss:  1.0346789315097384 val accuracy:  0.8319797551789078\n",
            "--------\n",
            "acc 0.8319797551789078\n",
            "Training..... epoch nr:  9\n",
            "--------\n",
            "train loss:  0.9913825253796668 val accuracy:  0.8303319209039548\n",
            "--------\n",
            "acc 0.8319797551789078\n",
            "Training..... epoch nr:  10\n",
            "--------\n",
            "train loss:  0.954040287524163 val accuracy:  0.83674670433145\n",
            "--------\n",
            "acc 0.83674670433145\n",
            "Training..... epoch nr:  11\n",
            "--------\n",
            "train loss:  0.921432346861021 val accuracy:  0.8380414312617702\n",
            "--------\n",
            "acc 0.8380414312617702\n",
            "Training..... epoch nr:  12\n",
            "--------\n",
            "train loss:  0.8928167579134991 val accuracy:  0.8409839924670434\n",
            "--------\n",
            "acc 0.8409839924670434\n",
            "Training..... epoch nr:  13\n",
            "--------\n",
            "train loss:  0.8673561959568409 val accuracy:  0.8417490583804144\n",
            "--------\n",
            "acc 0.8417490583804144\n",
            "Training..... epoch nr:  14\n",
            "--------\n",
            "train loss:  0.8455477301989822 val accuracy:  0.8435734463276836\n",
            "--------\n",
            "acc 0.8435734463276836\n",
            "Training..... epoch nr:  15\n",
            "--------\n",
            "train loss:  0.825202478704967 val accuracy:  0.841454802259887\n",
            "--------\n",
            "acc 0.8435734463276836\n",
            "Training..... epoch nr:  16\n",
            "--------\n",
            "train loss:  0.8069732998760454 val accuracy:  0.8417490583804144\n",
            "--------\n",
            "acc 0.8435734463276836\n",
            "Training..... epoch nr:  17\n",
            "--------\n",
            "train loss:  0.7902206173949621 val accuracy:  0.8542255178907722\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  18\n",
            "--------\n",
            "train loss:  0.7745841615541663 val accuracy:  0.8478107344632768\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  19\n",
            "--------\n",
            "train loss:  0.7610470297020424 val accuracy:  0.84439736346516\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  20\n",
            "--------\n",
            "train loss:  0.7478815547510828 val accuracy:  0.8539901129943502\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  21\n",
            "--------\n",
            "train loss:  0.7359174882257187 val accuracy:  0.8444562146892656\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  22\n",
            "--------\n",
            "train loss:  0.7249607683942622 val accuracy:  0.849988229755179\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  23\n",
            "--------\n",
            "train loss:  0.7146542509179681 val accuracy:  0.8483992467043314\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  24\n",
            "--------\n",
            "train loss:  0.7049462793625806 val accuracy:  0.8489289077212806\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  25\n",
            "--------\n",
            "train loss:  0.695715570990699 val accuracy:  0.8462806026365348\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  26\n",
            "--------\n",
            "train loss:  0.687250320677807 val accuracy:  0.8524011299435028\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  27\n",
            "--------\n",
            "train loss:  0.6794884567862778 val accuracy:  0.8502236346516008\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  28\n",
            "--------\n",
            "train loss:  0.6720663207444292 val accuracy:  0.8459863465160076\n",
            "--------\n",
            "acc 0.8542255178907722\n",
            "Training..... epoch nr:  29\n",
            "--------\n",
            "train loss:  0.6649078351444315 val accuracy:  0.8422787193973634\n",
            "--------\n",
            "train losses: 2.4278 / 1.9151 / 1.6348 / 1.4515 / 1.3227 / 1.2260 / 1.1484 / 1.0862 / 1.0347 / 0.9914 / 0.9540 / 0.9214 / 0.8928 / 0.8674 / 0.8455 / 0.8252 / 0.8070 / 0.7902 / 0.7746 / 0.7610 / 0.7479 / 0.7359 / 0.7250 / 0.7147 / 0.7049 / 0.6957 / 0.6873 / 0.6795 / 0.6721 / 0.6649\n",
            "val   losses: 0.3945 / 0.5578 / 0.4161 / 0.3838 / 0.3302 / 0.5408 / 0.3647 / 0.6740 / 0.2585 / 0.3683 / 0.6790 / 0.0980 / 0.3361 / 0.2441 / 0.2156 / 0.4685 / 0.2593 / 0.6006 / 0.2722 / 0.8707 / 0.4974 / 0.4690 / 0.1294 / 0.2947 / 0.1399 / 0.3200 / 0.2745 / 0.2232 / 0.5610 / 0.7474 / 0.9552 / 0.5497 / 0.4498 / 0.5466 / 0.5813 / 0.2580 / 0.7715 / 0.3592 / 0.3879 / 0.2769 / 0.4936 / 0.5239 / 0.4745 / 0.6280 / 0.2771 / 0.8161 / 0.2200 / 0.3350 / 0.6461 / 0.7180 / 0.8183 / 0.4106 / 0.3738 / 0.4900 / 0.1684 / 1.1720 / 0.6848 / 0.9733 / 1.3040\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "L7KhwZHXvnQc",
        "outputId": "b420b714-840c-431a-9e3e-13dd68779b0b"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(epoch_losses)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8dcnOwnZE8KWsAUXUFmMoIAIbQfRsaUd69aOta0dxqodO78u09m0v3Z+/bW/zrS1ra1Sa21nrNatrW3dWxQRQQOCrEIIW1iSQCAhJAGSfH5/3Ate4g1hyc1Jct/Px+M+7r3n+733fji5yZtzvt9zjrk7IiIiHSUEXYCIiPROCggREYlKASEiIlEpIEREJCoFhIiIRKWAEBGRqGIWEGZWbGYLzWydma01s7ui9JllZvVmtjJ8uzuiba6ZvWtmFWb2tVjVKSIi0SXF8L1bgS+5+wozywSWm9lL7r6uQ7/X3P2ayAVmlgjcB/wVUAW8ZWbPRHmtiIjESMy2INx9t7uvCD8+CKwHhp3iy6cAFe5e6e5HgMeAebGpVEREoonlFsRxZjYSmAQsi9J8mZmtAnYBX3b3tYSCZEdEnypgalefU1BQ4CNHjjzbckVE4sby5cv3unthtLaYB4SZDQSeAr7o7g0dmlcAI9y90cyuBn4HjD3N958PzAcoKSmhvLy8G6oWEYkPZrats7aYzmIys2RC4fCIuz/dsd3dG9y9Mfz4WSDZzAqAnUBxRNfh4WXv4+4L3L3M3csKC6OGoIiInIFYzmIy4OfAenf/Xid9Bof7YWZTwvXsA94CxprZKDNLAW4EnolVrSIi8n6x3MU0HbgZWG1mK8PL/gUoAXD3+4GPA583s1agGbjRQ6eXbTWzO4EXgETgofDYhIiI9BDrT6f7Lisrc41BiIicOjNb7u5l0dp0JLWIiESlgBARkagUECIiElXcB0TL0TYeeHUzizftDboUEZFeJe4DIiUxgQWLKnl6RVXQpYiI9CpxHxAJCca00gIWV+ylP83oEhE5W3EfEAAzSvOpOXiYiprGoEsREek1FBDAtDEFACyu0DiEiMgxCgigOC+dEfnpvF6xL+hSRER6DQVE2PTSApZW7qO1rT3oUkREegUFRNiM0gIaD7eyqqo+6FJERHoFBUTYZaPzMYPXNQ4hIgIoII7LzUhh/NAsDVSLiIQpICJMLy3g7e37aTrSGnQpIiKBU0BEmFFawNE2580tdUGXIiISOAVEhEtG5pGSlKBxCBERFBAnSEtOpGxELot1PISIiAKio+mlBazf3cDexsNBlyIiEigFRAfTS0On3ViyWVsRIhLfFBAdXDgsm8y0JJZoHEJE4pwCooPEBGPamHxe26TTf4tIfItZQJhZsZktNLN1ZrbWzO6K0ueTZvaOma02syVmNiGibWt4+UozK49VndHMKC1g54Fmttc19eTHioj0KkkxfO9W4EvuvsLMMoHlZvaSu6+L6LMFuMLd95vZVcACYGpE+2x37/F9PcfGIRZX7GVEfkZPf7yISK8Qsy0Id9/t7ivCjw8C64FhHfoscff94adLgeGxqud0jCrIYEh2mo6HEJG41iNjEGY2EpgELDtJt1uB5yKeO/CimS03s/knee/5ZlZuZuW1tbXdUS5mxvTSApZs3kd7u8YhRCQ+xTwgzGwg8BTwRXdv6KTPbEIB8U8Ri2e4+2TgKuAOM5sZ7bXuvsDdy9y9rLCwsNvqnlFawIGmo6zbHbVkEZF+L6YBYWbJhMLhEXd/upM+FwEPAvPc/fjBB+6+M3xfA/wWmBLLWjuaVpoP6DKkIhK/YjmLyYCfA+vd/Xud9CkBngZudveNEcszwgPbmFkGMAdYE6taoxmUmca5RZkahxCRuBXLWUzTgZuB1Wa2MrzsX4ASAHe/H7gbyAd+EsoTWt29DCgCfhtelgT82t2fj2GtUU0rzefXy7bTcrSNtOTEnv54EZFAxSwg3H0xYF30+RzwuSjLK4EJ739Fz5pRWsAvXt/Kim37mRae+ioiEi90JPVJTB2dT2KC8fpm7WYSkfijgDiJgalJTCrO0em/RSQuKSC6ML20gNVVB6hvOhp0KSIiPUoB0YUZYwtod3ijUlsRIhJfFBBdmDA8h/SURE13FZG4o4DoQkpSAlNH5SkgRCTuKCBOwfTSAir3HmLXgeagSxER6TEKiFMwY2zoGAhtRYhIPFFAnIJzizIpGJiigBCRuKKAOAVmxrQxBSyu2KfLkIpI3FBAnKIZpQXsbTzMxurGoEsREekRCohTNH3se5chFRGJBwqIUzQsZwCjCjJYooAQkTihgDgN00vzWVq5j6Nt7UGXIiIScwqI0zCjtIBDR9pYteNA0KWIiMScAuI0XDo6HzONQ4hIfFBAnIac9BQuHJat4yFEJC4oIE7T9NIC3t5+gEOHW4MuRUQkphQQp2lGaQGt7c6bW+qCLkVEJKYUEKfp4hG5pCYlaBxCRPo9BcRpSktOpGxkrsYhRKTfi1lAmFmxmS00s3VmttbM7orSx8zsh2ZWYWbvmNnkiLZbzGxT+HZLrOo8E9NLC9iw5yA1B1uCLkVEJGZiuQXRCnzJ3ccBlwJ3mNm4Dn2uAsaGb/OBnwKYWR5wDzAVmALcY2a5Maz1tMwoDZ12443NugypiPRfMQsId9/t7ivCjw8C64FhHbrNA37lIUuBHDMbAlwJvOTude6+H3gJmBurWk/X+KHZZA9I1m4mEenXemQMwsxGApOAZR2ahgE7Ip5XhZd1tjzae883s3IzK6+tre2ukk8qMcGYNiafxZv26vTfItJvxTwgzGwg8BTwRXdv6O73d/cF7l7m7mWFhYXd/fadml5awK76Frbua+qxzxQR6UkxDQgzSyYUDo+4+9NRuuwEiiOeDw8v62x5rzG9VKf/FpH+LZazmAz4ObDe3b/XSbdngE+FZzNdCtS7+27gBWCOmeWGB6fnhJf1GiPz0xmWM4DXNykgRKR/Sorhe08HbgZWm9nK8LJ/AUoA3P1+4FngaqACaAI+E26rM7NvAm+FX/cNd+9Vhy6bGdNL83l+zR7a2p3EBAu6JBGRbhWzgHD3xcBJ/2p6aIT3jk7aHgIeikFp3WZ6aQGPl1exdlc9Fw3PCbocEZFupSOpz8K0MRqHEJH+SwFxFgozUzlvcKaOhxCRfkkBcZamlxbw1tb9tBxtC7oUEZFupYA4SzNKCzjS2k751v1BlyIi0q0UEGdpyqg8khONP2+oDroUEZFupYA4SxmpSVx94RAef2sHB5qOBF2OiEi3UUB0g8/PGsOhI238csm2oEsREek2CohucN7gLD50fhG/WLJF16oWkX5DAdFNbp89hgNNR3n0ze1BlyIi0i0UEN1kckku08bks2BRJYdbNeVVRPo+BUQ3umN2KTUHD/PU8l514lkRkTOigOhG08bkM6E4h/tf3UxrW3vQ5YiInBUFRDcyM+6YNYbtdU388Z3dQZcjInJWFBDd7EPnF3FuUSY/eaWC9nZdjlRE+i4FRDdLSDBunz2GjdWNvLxeR1eLSN+lgIiBv75wCCV56dz3ymZCl7wQEel7FBAxkJSYwG1XjGHVjgMs2bwv6HJERM6IAiJGrr14GIMyU7lvYUXQpYiInBEFRIykJiUyf+Zolmzex4rtOhW4iPQ9CogYumlKCTnpyfxk4eagSxEROW0KiBjKSE3iM9NG8fL6ajbsaQi6HBGR0xKzgDCzh8ysxszWdNL+FTNbGb6tMbM2M8sLt201s9XhtvJY1dgTbpk2goyURH76irYiRKRvieUWxMPA3M4a3f277j7R3ScC/wy86u51EV1mh9vLYlhjzOWkp/C3l43gD6t2sXXvoaDLERE5ZTELCHdfBNR12THkJuDRWNUStFtnjCIpMYEHFmkrQkT6jsDHIMwsndCWxlMRix140cyWm9n8Ll4/38zKzay8trY2lqWesUGZadxQVsyTy6vYU98SdDkiIqck8IAAPgy83mH30gx3nwxcBdxhZjM7e7G7L3D3MncvKywsjHWtZ2z+zNG0O/zstcqgSxEROSW9ISBupMPuJXffGb6vAX4LTAmgrm5VnJfOvIlD+fWy7dQdOhJ0OSIiXQo0IMwsG7gC+H3Esgwzyzz2GJgDRJ0J1dfcPmsMLa1tPPz6lqBLERHpUiynuT4KvAGca2ZVZnarmd1mZrdFdPsY8KK7R07vKQIWm9kq4E3gT+7+fKzq7EmlgzK5ctxgHl6ylYMtR4MuR0TkpJJi9cbuftMp9HmY0HTYyGWVwITYVBW822eP4fm1e3hk2XZuu2JM0OWIiHSqN4xBxJWLhudw+dgCHnxtCy1H24IuR0SkUwqIANwxu5S9jYd5onxH0KWIiHRKARGAqaPyKBuRy/2vVnK0rT3ockREolJABMDMuGN2KTsPNPP7lbuCLkdEJCoFREBmnVvI+UOy+MkrFbS167KkItL7KCACEtqKGENl7SFeXLsn6HJERN5HARGgqy4YwqiCDO57pQJ3bUWISO+igAhQYoLx+SvGsGZnA69u7J0nGhSR+KWACNhHJw1jeO4A7nlmrY6uFpFeRQERsJSkBL53/UR21DXxb79bo11NItJrnDQgzCzbzL5tZhvMrM7M9pnZ+vCynJ4qsr+bMiqPf/zQOfx+5S6eKK8KuhwREaDrLYjHgf3ALHfPc/d8YHZ42eOxLi6e3D67lGlj8rn7mTVU1BwMuhwRkS4DYqS7f8fdj8/DdPc97v4dYERsS4sviQnGD26YSEZKEnc88rbO0yQigesqILaZ2VfNrOjYAjMrMrN/AnQioW42KCuN/7p+Au9WH+Qbf1wXdDkiEue6CogbgHzgVTPbb2Z1wCtAHnB9jGuLS7POHcTfXzGaXy/bzp/e2R10OSISx04aEO6+H/gFcCdQHB6HON/d/4l+cBnQ3urLc85lYnEOX3vqHXbUNQVdjojEqa5mMf0DocuB3gmsMbN5Ec3fimVh8Sw5MYEf3TQJDO589G2OtOqMryLS87raxfR3wMXu/lFgFvDvZnZXuM1iWVi8K85L5zvXXsSqHQf4rxffDbocEYlDXQVEgrs3Arj7VkIhcZWZfQ8FRMxdfeEQPjm1hAcWVbLw3ZqgyxGRONNVQFSb2cRjT8JhcQ1QAFwYy8Ik5N+vGcd5gzP50uOrqG5oCbocEYkjXQXEp4ATzkXt7q3u/ilg5sleaGYPmVmNma3ppH2WmdWb2crw7e6Itrlm9q6ZVZjZ107x39IvpSUn8uNPTKL5SBtffGylrh0hIj2mq1lMVZEHyXVoe72L934YmNtFn9fcfWL49g0AM0sE7gOuAsYBN5nZuC7ep18rHZTJ/543njcq93HfwoqgyxGROBGzk/W5+yKg7gxeOgWocPdKdz8CPAbM6+I1/d51Fw/noxOH8oOXN7Kscl/Q5YhIHAj6bK6XmdkqM3vOzMaHlw3jxKO0q8LL4pqZ8R8fu5CSvHTuemwl+w8dCbokEennggyIFcAId58A/Aj43Zm8iZnNN7NyMyuvre3fF90ZmJrEjz8xmbpDR/jyE6t0anARianAAsLdGyKm0D4LJJtZAbATKI7oOjy8rLP3WeDuZe5eVlhYGNOae4MLhmXzz1efx5831PDQ61uDLkdE+rHAAsLMBpuZhR9PCdeyD3gLGGtmo8wsBbgReCaoOnujT08byYfOL+Lbz61ndVV90OWISD8Vs4Aws0eBN4BzzazKzG41s9vM7LZwl48TOn3HKuCHwI0e0kro1B4vAOuBx919bazq7IvMjO9+/CIKBqZy56MrdKlSEYkJ60/7scvKyry8vDzoMnrMW1vruOGBN7jmoqHce+NEwhtkIiKnzMyWu3tZtLagZzHJWbhkZOhSpc+s2sV/L90WdDki0s8kBV2AnJ3bZ5fy9o4D3P37tbS3O5+ePirokkSkn9AWRB+XmGD89G8nM2dcEV//wzodaS0i3UYB0Q+kJiVy3ycnM2/iUL77wrt894UNOkZCRM6adjH1E8mJCXzv+okMSE7kvoWbaTrSxt3XjNPAtYicMQVEP5KYYPzfv7mQASmJ/OL1rTQfaeP/fOxCEhMUEiJy+hQQ/YyZcfc148hISeLHCytoPtrGf143geRE7U0UkdOjgOiHzIwvX3kuA1IS+e4L79J8pI0ffWISqUmJQZcmIn2I/lvZj90xu5R7PjyOF9dV83e/Wk7zkbagSxKRPkQB0c99ZvoovnPthby2qZZbfvEmjYdbgy5JRPoIBUQcuOGSEn5ww0SWb9vPJx9cRn2Tzt0kIl1TQMSJeROH8dNPTmb9rgZu/NlS9jYeDrokEenlFBBxZM74wTx4Sxlb9jZy/QNvsKe+JeiSRKQXU0DEmZnnFPKrz06lpuEw1z2whB11TUGXJCK9lAIiDk0Zlccjn5tKQ3Mr193/BptrG4MuSUR6IQVEnJpQnMNj8y+ltb2dGx54gzU7dWU6ETmRAiKOnT8ki9/8/WUkJybwNz9dwi+XbNVJ/kTkOAVEnBtTOJA/fGEG08fkc88za/ncL8vZpxlOIoICQoCCgak89OlLuOfD43ht017m3vsar22qDbosEQmYAkKA0PmbPjN9FL+7YzrZA5K5+edv8q1n13OktT3o0kQkIAoIOcG4oVn84c4ZfGJqCQsWVXLtT5dQqVlOInEpZgFhZg+ZWY2Zremk/ZNm9o6ZrTazJWY2IaJta3j5SjMrj1WNEt2AlES+9bELuf9vL2bH/iau+dFiHi/foQFskTgTyy2Ih4G5J2nfAlzh7hcC3wQWdGif7e4T3b0sRvVJF+ZeMJjn7rqci4Zn89Un3+ELj75NfbPO4yQSL2IWEO6+CKg7SfsSd98ffroUGB6rWuTMDckewCOfu5SvXHkuz63Zw9X3vkb51k5/rCLSj/SWMYhbgecinjvwopktN7P5AdUkYYkJxh2zS3nytstISIDrH3iDe1/eRGubBrBF+rPAA8LMZhMKiH+KWDzD3ScDVwF3mNnMk7x+vpmVm1l5ba2mZsbSpJJcnv2Hy/nIhKF8/+WN3PSzpew80Bx0WSISI4EGhJldBDwIzHP3fceWu/vO8H0N8FtgSmfv4e4L3L3M3csKCwtjXXLcy0xL5gc3TuL7N0xg3a4GrvrBIp5dvTvoskQkBgILCDMrAZ4Gbnb3jRHLM8ws89hjYA4QdSaUBOdjk4bz7F2XM6pwILc/soJ/ePRtbU2I9DNJsXpjM3sUmAUUmFkVcA+QDODu9wN3A/nAT8wMoDU8Y6kI+G14WRLwa3d/PlZ1ypkbkZ/Bk7ddxo//UsH9r27mhbV7mD9zNLddMYaM1Jh9tUSkh1h/mtteVlbm5eU6bCIIOw80853nNvDMql0UZqbylTnncu3Fw0lMsKBLE5GTMLPlnR1OEPggtfQPw3IG8MObJvH07dMYnjuArz71Dh/+0WLe2Lyv6xeLSK+kgJBuNbkkl6c/P40f3jSJ+uaj3PSzpcz/VTlb9h4KujQROU0KCOl2ZsZHJgzlz1+6gq9ceS6vV+xlzvdf5Zt/XEd9k47EFukrFBASM2nJidwxu5SFX5nFtZOH89DrW7jiPxfy8OtbOKqD7ER6PQWExNygzDS+fe1F/OkLlzN+aBZf/8M65v5gEX/ZUK0TAIr0YgoI6THjhmbxP7dO5cFPleEOn324nJt//iYb9jQEXZqIRKGAkB5lZnxoXBEv/ONM7vnwOFbvrOfqe1/jfz2+knf3HAy6PBGJoOMgJFAHmo7w479U8Miy7TQfbeMD5w1i/szRTB2VR/hgSRGJoZMdB6GAkF5h/6Ej/PfSbfxyyVb2HTrChOIcbps5mjnjB+tgO5EYUkBIn9FytI0nl1fxs9cq2baviZH56Xzu8tF8/OLhpCUnBl2eSL+jgJA+p63deXHtHu5/dTOrqurJz0jh09NGcvNlI8hJTwm6PJF+QwEhfZa7s2xLHQ+8upmF79YyIDmRGy4p5tYZoyjOSw+6PJE+TwEh/cK7ew6yYFElv1+5EweuuWgI82eOZvzQ7KBLE+mzFBDSr+yub+ahxVt49M0dNB5u5fKxBXxm+khmji0kKVEzt0VOhwJC+qX65qP8etl2Hnp9C7UHD1OUlcq1k4dzXVkxowoygi5PpE9QQEi/dqS1nb9sqOGJ8h0sfLeGdocpI/O4rmw4V184RBcvEjkJBYTEjeqGFp5esZMnyndQufcQ6SmJXHPREK4vK+biEbk6+E6kAwWExB13Z8X2/Tz+VhV/fGcXh460Mbogg+vKirl28jAGZaUFXaJIr6CAkLh26HArz67ezRPlVby5tY7EBGPWOYVcV1bMB84bREqSBrYlfikgRMIqaxt5cnkVT62oorrhMPkZKXx00jA+NmkY44dmaReUxB0FhEgHrW3tvFaxlyfKd/DSumqOtjnFeQOYO34wcy8YzKTiXBJ0DiiJA4EFhJk9BFwD1Lj7BVHaDbgXuBpoAj7t7ivCbbcA/xbu+h/u/suuPk8BIWei7tARXl5XzfNr97B4016OtLVTmJnKleOLmDt+CFNH55Gs4yuknwoyIGYCjcCvOgmIq4EvEAqIqcC97j7VzPKAcqAMcGA5cLG77z/Z5ykg5Gw1tBxl4YYaXli7h4Ubamk+2kb2gGQ+dH4Rcy8YzOVjC3TSQOlXThYQMZ0g7u6LzGzkSbrMIxQeDiw1sxwzGwLMAl5y9zoAM3sJmAs8Gst6RbLSkpk3cRjzJg6j5WgbizbW8vzaPby0bg9PragiPSWR2ecNYu74wcw+bxADdYyF9GNBf7uHATsinleFl3W2/H3MbD4wH6CkpCQ2VUpcSktOZM74wcwZP5ijbe0srdzHc2v28OLaav70zm5SEhO4fGwBV14wmA+eN4j8galBlyzSrYIOiLPm7guABRDaxRRwOdJPJScmcPnYQi4fW8g3513Aiu37eX7NHp5fs4c/b6jBDC4Yms3Mcwq4fGwhk0tyNX1W+rygA2InUBzxfHh42U5Cu5kil7/SY1WJnERignHJyDwuGZnHv/31+azd1cDCDTUs2lTL/a9Wct/CzWSkJHLZmAKuOKeAmecUMiJf54aSvifogHgGuNPMHiM0SF3v7rvN7AXgW2aWG+43B/jnoIoU6YyZccGwbC4Yls0XPjiWhpajLKnYx6JNtSzaWMvL66sBGJGfzuVjC5g5tpBppQUau5A+IabfUjN7lNCWQIGZVQH3AMkA7n4/8CyhGUwVhKa5fibcVmdm3wTeCr/VN44NWIv0Zllpycy9IHQshbuzdV8TizaGwuLpFTv5n6XbSUowJo/I5YpzCpk5tpDxQ7N0zIX0SjpQTqSHHG5tY/m2/SzauJfXNtWydlcDAHkZKUwbk8/UUXlMGZXP2EEDFRjSY3QktUgvVHvwMIsralm0cS9vbN7HnoYWAHLTk7lkZB5TRuVx6eh8zh+SRaICQ2JEASHSy7k7O+qaWbZlH29uqWPZljq21zUBkJmaxMUjc5k6Kp8po/K4cFi2ZkhJtwnsQDkROTVmRkl+OiX56VxXFprYt7u+mTe31B0PjFfe3QDAgOREJo/IYcrIUGBMKsnR0d0SE9qCEOkj9jYepnxrHUsrQ6Gxfk8D7pCSmMAFw7KYWJzLhOJsJhbnUJKXrjPTyinRLiaRfqi++SjLt9WxrLKOFdv3s3pnPS1H24HQOMaE4hwmFucwoTiHCcNzyMtICbhi6Y20i0mkH8oekMwHziviA+cVAaFTmL9bfZBVO+pZuWM/q3bU8+rGTRz7P+CI/HQmDH8vNMYPzdKuKTkpbUGI9GONh1tZXVXPqqoDrNx+gFVVB9hdH5otlZRgnD8kiwnF2Vw0LIdxQ7MYWzSQ1CSFRjzRLiYROa66oYWVOw6wcscBVu04wDtV9TQebgVCoVE6aCDjhmQxbmgW44Zkcf6QLHK1e6rfUkCISKfa251tdU2s29XAut314fsGqhsOH+8zJDvthNAYNzSL4tx0HdDXD2gMQkQ6lZBgjCrIYFRBBn990ZDjy/c2Hmb97gbW7244HhqvbKylrT30n8qBqUmcNziTcUOzOG9waPfUOYMyyU5PDuqfIt1MASEiURUMTD1+ivNjWo62sbH6IOt2hYNjdwNPr9hJ4+Ftx/sUZqYydtDA0K0o8/i9ZlH1PQoIETllacmJXDQ8h4uG5xxf1t7u7DzQTEVNI5tqDrKpupGNNY08ubyKQ0fajvfLz0hhbNFAxg7KZGzRQEoHDeScokzyM1J0zEYvpYAQkbOSkGAU56VTnJfO7PMGHV/u7uyub2FTTSObqkPBsanmIL9buZODLa3H++WmJ1M6aCCjCwYyujCD0YWh+5K8dJITdUqRICkgRCQmzIyhOQMYmjOAK855bzeVu1Nz8HBoS6P6IJtqDrK55hB/3lDNb8qPHO+XlGCU5KUzujCDMYUR4VGQQZ62OnqEAkJEepSZUZSVRlFWGjPGFpzQVt98lMraRiprD1G5N3S/ubaRRRv3cqSt/Xi/7AHJocA4ttVRkEFJfjoj8jN0MaZupDUpIr1G9oBkJpXkMqkk94Tlbe3Ozv3NbA6HxrEQWVxRy1Mrqk7om5+REgqLvHRK8kO7qkaEnxdmpmrL4zQoIESk10tMeO9st7PPPbHtYMtRtu1rYntdU/j+ENv2NfHW1v08s2oX7RGHeg1ITqQkL/14gIzID4VIcW5oV5hOPXIiBYSI9GmZacnHrwve0ZHWdqr2N7Gtront+94LkK17D7FoYy2HW9tP6F8wMJVhuQMYlpPGsJwBoVtuOkNz0hiek07WgKS42gJRQIhIv5WSlBCeFTXwfW3t7U5t42G27WtiR10TOw80s+tAMzsPNLNh90H+vL7mfQEyMDWJYTkDGJqTFg6S9OOBMjh7AEWZqST1o5lXCggRiUsJCe8Nlk8Zlfe+dndn36Ej7NzffDw8qsKPd+5v5u0dBzjQdPTE97TQgYKDswcwNDuNwdlpDM0ewODsNIZkpzEkZwCDMlP7zPRdBYSISBRmRsHAVAoGpjKhOCdqn8bDrce3OvbUt7C7voU99c3Hj/9YtLH2hIMF4cQQGZKVxpCcUHgUZaUxKDMUKkVZqaSnBP/nOaYVmNlc4F4gEXjQ3b/dof37wOzw03RgkLvnhA5MMMYAAAcUSURBVNvagNXhtu3u/pFY1ioicroGpiZxTlEm5xRlRm13dw4ebmX3gRZ2h4Njd30Luw80s6ehhYraRl7b9P4QAchMSwpv4aQe39IpykxlcHYag7LSGJyVRmGMt0ZiFhBmlgjcB/wVUAW8ZWbPuPu6Y33c/R8j+n8BmBTxFs3uPjFW9YmIxJqZkZWWTNbgZM4dfPIQqWloobrhMNUNLexpaKGm4TB76luoPtjCsso6qhtaaG33Du8fmtY7qiCDJ26b1u31x3ILYgpQ4e6VAGb2GDAPWNdJ/5uAe2JYj4hIr3M8RNKSKR0UPUQgNKhe13SE6oaW8O3w8cexEsuAGAbsiHheBUyN1tHMRgCjgL9ELE4zs3KgFfi2u/+uk9fOB+YDlJSUdEPZIiK9T0LCe2Mi44e+f0pvTD6zRz6lazcCT7p75I64EeGLWHwC+IGZjYn2Qndf4O5l7l5WWFgYrYuIiJyBWAbETqA44vnw8LJobgQejVzg7jvD95XAK5w4PiEiIjEWy4B4CxhrZqPMLIVQCDzTsZOZnQfkAm9ELMs1s9Tw4wJgOp2PXYiISAzEbAzC3VvN7E7gBULTXB9y97Vm9g2g3N2PhcWNwGN+4sWxzwceMLN2QiH27cjZTyIiEnt24t/lvq2srMzLy8uDLkNEpM8ws+Xh8d736S2D1CIi0ssoIEREJCoFhIiIRNWvxiDMrBbYdoYvLwD2dmM53U31nR3Vd3ZU39npzfWNcPeoB5H1q4A4G2ZW3tlATW+g+s6O6js7qu/s9Pb6OqNdTCIiEpUCQkREolJAvGdB0AV0QfWdHdV3dlTf2ent9UWlMQgREYlKWxAiIhJV3AWEmc01s3fNrMLMvhalPdXMfhNuX2ZmI3uwtmIzW2hm68xsrZndFaXPLDOrN7OV4dvdPVVf+PO3mtnq8Ge/77wmFvLD8Pp7x8wm92Bt50asl5Vm1mBmX+zQp0fXn5k9ZGY1ZrYmYlmemb1kZpvC97mdvPaWcJ9NZnZLD9b3XTPbEP75/dbMol6QuavvQgzr+7qZ7Yz4GV7dyWtP+rsew/p+E1HbVjNb2clrY77+zpq7x82N0EkDNwOjgRRgFTCuQ5/bgfvDj28EftOD9Q0BJocfZwIbo9Q3C/hjgOtwK1BwkvargecAAy4FlgX4s95DaI53YOsPmAlMBtZELPt/wNfCj78GfCfK6/KAyvB9bvhxbg/VNwdICj/+TrT6TuW7EMP6vg58+RR+/if9XY9VfR3a/wu4O6j1d7a3eNuCOH4ZVHc/Ahy7DGqkecAvw4+fBD5oZtYTxbn7bndfEX58EFhP6Mp8fck84FceshTIMbMhAdTxQWCzu5/pgZPdwt0XAXUdFkd+x34JfDTKS68EXnL3OnffD7wEzO2J+tz9RXdvDT9dSuhaLoHoZP2dilP5XT9rJ6sv/Hfjejpc66YvibeAiHYZ1I5/gI/3Cf+S1AP5PVJdhPCurUnAsijNl5nZKjN7zszG92hh4MCLZrY8fLnXjk5lHfeE912EKkKQ6w+gyN13hx/vAYqi9Okt6/GzhLYIo+nquxBLd4Z3gT3UyS663rD+Lgeq3X1TJ+1Brr9TEm8B0SeY2UDgKeCL7t7QoXkFod0mE4AfAVGv1R1DM9x9MnAVcIeZzezhz++ShS5Q9RHgiSjNQa+/E3hoX0OvnEpoZv9K6Jrwj3TSJajvwk+BMcBEYDeh3Ti90U2cfOuh1/8uxVtAnMplUI/3MbMkIBvY1yPVhT4zmVA4POLuT3dsd/cGd28MP34WSLbQVfd6hL93Kdga4LeENuUjnc6lZmPlKmCFu1d3bAh6/YVVH9vtFr6vidIn0PVoZp8GrgE+GQ6x9zmF70JMuHu1u7e5ezvws04+N+j1lwT8DfCbzvoEtf5OR7wFxKlcBvUZ4NiMkY8Df+nsF6S7hfdZ/hxY7+7f66TP4GNjImY2hdDPsEcCzMwyzCzz2GNCg5lrOnR7BvhUeDbTpUB9xO6UntLp/9yCXH8RIr9jtwC/j9LnBWCOhS6/m0toXb/QE8WZ2Vzgq8BH3L2pkz6n8l2IVX2RY1of6+RzT+mSxzH0IWCDu1dFawxy/Z2WoEfJe/pGaJbNRkIzHP41vOwbhH4ZANII7ZqoAN4ERvdgbTMI7W54B1gZvl0N3AbcFu5zJ7CW0KyMpcC0HqxvdPhzV4VrOLb+Iusz4L7w+l0NlPXwzzeD0B/87Ihlga0/QkG1GzhKaD/4rYTGtP4MbAJeBvLCfcuAByNe+9nw97AC+EwP1ldBaP/9se/gsVl9Q4FnT/Zd6KH6/jv83XqH0B/9IR3rCz9/3+96T9QXXv7wse9cRN8eX39ne9OR1CIiElW87WISEZFTpIAQEZGoFBAiIhKVAkJERKJSQIiISFQKCBERiUoBISIiUSkgREQkqv8P0uBaN8J3S9wAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8v2EgUGvo_m",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "d141ae2e-807d-4113-c039-27bf142127b5"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(val_accs)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxddZ3/8dcna9t0TZOW7ju0ZWmBUJFNBIHCoK2Og606LCIMOmUU/SFldJAfyoyOzjDqAxcQLAhSGBQsChaQgoAsTaWUtnS56UL33DTdkjbLvfczf9yTcknTLE1ObpL7fj4e95F7vud7zv2c0+R+er7f7zlfc3dERERaKyvdAYiISPeixCEiIm2ixCEiIm2ixCEiIm2ixCEiIm2Sk+4AOkNRUZGPHTs23WGIiHQry5Ytq3D34sblGZE4xo4dS2lpabrDEBHpVsxsc1PlaqoSEZE2UeIQEZE2UeIQEZE2UeIQEZE2CTVxmNlMM1trZhEzm9/E+tFmtsTM3jKzFWZ2WVA+1swOmdny4PXzlG1ON7N3gn3+2MwszGMQEZEPCi1xmFk2cDdwKTAVmGtmUxtV+xbwmLufCswBfpqyrszdpwevG1LKfwZcB0wKXjPDOgYRETlSmFccM4CIu29w9zpgITCrUR0H+gfvBwDbm9uhmQ0D+rv76558rO+DwOyODVtERJoTZuIYAWxJWd4alKW6Hfi8mW0FngZuTFk3LmjCesnMzk3Z59YW9gmAmV1vZqVmVhqNRttxGCIi3c+7O/Zzx1OricUTHb7vdN8AOBdY4O7/ZWYfBn5tZicBO4DR7r7bzE4HnjSzE9uyY3e/B7gHoKSkRJOOiEiPl0g4L62Pct/LG3klUkHv3Gw+eeoITh45oEM/J8zEsQ0YlbI8MihLdS1BH4W7v2ZmvYAidy8HaoPyZWZWBhwfbD+yhX2KiGSUmvo4T7y1jfte2UikvIqh/fP5xswT+OyM0Qzsk9fhnxdm4lgKTDKzcSS/3OcAn21U5z3gQmCBmU0BegFRMysGKt09bmbjSXaCb3D3SjPbb2ZnAm8AVwI/CfEYRES6rOiBWn79+mYeen0zldV1nDi8P3d9Zhp/d/Jw8nLC64kILXG4e8zM5gGLgWzgfndfZWZ3AKXuvgj4OnCvmd1EsqP8and3MzsPuMPM6oEEcIO7Vwa7/jKwAOgNPBO8RKSLcXfiCScnW7eLdbS1Ow9w3ysbePKt7dTFE3xsyhCuPWc8Z44vpDPuULBMmHO8pKTE9ZBDkfCVH6jhr5HdvBqp4NVIBRVVdXzkhGI+Pm04H5syhD556e5W7b7cnb+sr+CXL2/g5fUV9MrN4tOnj+QLZ49jfHHfUD7TzJa5e0njcv0risgx219TzxsbKg8nivXlVQAM7JPLh8cPprhfPotX7eS51bvok5fNx6YM5RPThnPe8cWhNqX0JDX1cX6/PNl/sW5XFUP65XPzJcn+i0EFHd9/0Rq64hCRVqupj/O39/bw18huXolU8M62fcQTTq/cLM4YW8jZE4s4e0IRU4f3Jzsr2WQSTzhvbqxk0dvbeWblDvYerKd/rxwuPWkYn5g+nDPHDz5ct7uJxRNUHqyjsrqO3VV17K6uY3dVLZXVdVRU1bG/pp5sM3KyjdysLLKzjdwsIyc7i5ysZHlOVsP7LHKzjeyG91nG9r2HePiN99hdXceUYf257txxXH5KuP0XqY52xaHEIdLFxBNOXSxBXSyB4/TNz0lbP0E84azavo9Xg+anpZsqqY0lyM4ypo0cwNkTizhrQhGnjRlIfk52i/urjyd4ZX0Fi97ezrOrdlJdF6eobz6XnzKMj08bzmmjBx5zG3084ezcX8N7uw+yZc9BtlQepKKqliyzw1/MLX1Z52Ql6+ZmZ5GdZTiwp/qDCSGZIGrZXV3H3oP1TcaSZVBYkEf/Xrkk3KmPJ/t7YokE9XEnFk8QSzixRLK8ORdOHsK1547jw+MHd0r/RSolDiUO6UTuzp9W7uT5d8upicWpiyWojSWoi8WDn4mUsgS1KXViTXyR9M3PoX+vHPr3zqVfrxz698qlf+/cw2XJ5dTyZL383CwO1sU5WBunui7GwboYVbVxDtbGqK57/2d1bSy5/nC9ZNn2vYfYXxMD4ISh/Thr4mDOnlDEh8YX0q9XbrvO0aG6OEvWlrNo+XZeWFtOXSzByEG9+fi04Xz8lOFMGdbviC/KfQfrea8ymRjeq0y+tgSvbXsPUR9//9xlGQzum4+nfHHXB1/YLX1ZN2YGg/rkUViQx+CCPAb3zWNwQT6FBXkU9c2jsCA/KMtjcN98BvbOJauVV1HuyQQSizv1iQTx4Gcs7uTnZDG4b36bYu1IShxKHNJJlm/Zy3f/sJrSzXsYXJDHgN655OVkkZ+TRX5ONvm5WeRlZ73/Myf78Pq8nA8uAxyoibG/pp79h+rZX1Ofsvx+eRu/Bz8gLyeLgrxs+uTlUJD//s+CvBwG983nzPGFfHjCYIb069VBZ+hI+2vqeXbVLp56ezuvRCqIJ5yJQ/py1oTBRA/UHk4QDUmswaA+uYwq7MOowj6MLuzDqEHBz8LeDB/Ym9yjXKmlflnHgi/pWKLR++CO60EFeQzsnZuRo8OUOJQ4JGTb9h7iB39aw5PLt1PUN4+vX3wCV5SMCr393t2prosfTiz7D8U4UJN8f6gucUQyOLycl0PvvOwu10m9u6qWp1fu5Knl23ln2z6GDezVKCkkE8Oowj70b+dVjzRPiUOJQ0JSVRvj5y+Wce/LGwD44rnj+NL5E+mbr0GL0r1pOK50O1v3HGT19v2cMbYwbcMOmxNPOP9buoUfPruOiqpaZk8fzs0zJzNiYO90hyYSKiUO6ZJeWV/Blx5exoGaGFkG00YN5LxJxXzkhGKmjRyY9uGbL6+Pcucf32XNzgOUjBnEL68qYfqogWmNSaSzKHFIl/PQ65v59qJVTCzuy61zJ7N8y15eXBvlxy+s50d/Xs/APrmcM7GIjxxfzEeOL2ZI//A6bRuLlB/gzj++y5K1UUYV9uannzuNS086rtOHSYqkk/o4pMuIxRN894/vsuCvm7hg8hB+PPfUD/QT7Kmu4+VIBS+tjfLSuigVVbUATBnW/3ASOX3MoFA6e3dX1fKjP6/n4Tfeo09uNjdeOJGrzhrbqnsXRLordY4rcXRp+2vqufE3b/HSuihfPGcct142pdnmqETCeXfnfl5aF+WltVGWbd5DLOEU5GVzVsrVyKjCPu2KqzYW54G/buInL0Q4WBfncx8azVcunJTWsfUinUWJQ4mjy3pv90GufWApGyuq+c7sk5g7Y3Sb93Ggpp6/lu0+nEi27T0EQFHfPPKys468azjbGt05nBU8CiJ1fRZvbtrNlspDXDB5CP962WQmDunX0Ycv0mVpVJV0SW9urOSGh5YRTzgPXjuDsyYUHdN++vXK5ZITj+OSE4/D3SmLVvPSuiiR8gMffMRD/P3HPjTcSVxTnyCWiCfrNNwQFtQd0j+ff//kyZw7qbiDj1yk+1LikLR5fNlWbv3dCkYN6sN9V5/BuKKCDtmvmTFxSF8mDgnnUdMimU6JQzpdIuH84Nm1/OzFMs6eOJiffvZ0BvTRHcAi3UWozxows5lmttbMImY2v4n1o81siZm9ZWYrzOyyoPwiM1tmZu8EPy9I2ebFYJ/Lg9eQMI9BOtbBuhhfengZP3uxjM9+aDQLrpmhpCHSzYR2xWFm2cDdwEXAVmCpmS1y99Up1b4FPObuPzOzqcDTwFigAvi4u283s5NITj87ImW7z7m7eru7mR37DnHtglLW7NzPbZdP5Zqzx+r+B5FuKMymqhlAxN03AJjZQmAWkJo4HOgfvB8AbAdw97dS6qwCeptZvrvXhhivhOjtLXv54oOlHKqLc9/VZ/DRE3ShKNJdhdlUNQLYkrK8lQ9eNQDcDnzezLaSvNq4sYn9/D3wt0ZJ41dBM9W/2VH+y2pm15tZqZmVRqPRYz4Iab8/rNjOFb94jfycLH735bOUNES6uXQ/T3kusMDdRwKXAb82s8MxmdmJwPeBf0rZ5nPufjJwbvD6x6Z27O73uHuJu5cUF2soZTq4Oz96fj3zfvMWJ48YwO//+WyOH6r7IES6uzCbqrYBo1KWRwZlqa4FZgK4+2tm1gsoAsrNbCTwBHClu5c1bODu24KfB8zsNySbxB4M7SikTRIJZ8W2fbywppznVu/i3R37+dRpI/iPT52sx3OI9BBhJo6lwCQzG0cyYcwBPtuoznvAhcACM5sC9AKiZjYQ+CMw391fbahsZjnAQHevMLNc4HLg+RCPQVrhQE09r6yv4IU15SxZGw3meYbTRg/izk+exGdnjFYnuEgPElricPeYmc0jOSIqG7jf3VeZ2R1AqbsvAr4O3GtmN5HsKL/a3T3YbiJwm5ndFuzyYqAaWBwkjWySSePesI5Bjm5TRTV/XlPOkjXlvLFxN/Vxp3+vHD5ywhAunDyEjxxf3CXn0BCR9tOzqqRV6uMJlm6qZMmacv68ppwN0WoAJg7py4WTh3DB5CGcPmZQRs7LLNJT6VlV0mbVtTH+tHInL6wt5y9roxyojZGXncWHxhdy5ZljuGDyUEYPbt/TZ0Wk+1HikCbtr6lnzi9eZ/WO/RT3y+eyk4dxwZQhnDOxiALNpS2S0fQNIEeoqY9z3QOlrNt1gJ9//nQunjqUrDRP1SoiXYcSh3xAPOF8deFy3thYyY/mTGfmScelOyQR6WLUkymHuTvfenIlf1q1k9sun8qs6Y1v9BcRUeKQFHc9t45H3nyPL58/gS+cMy7d4YhIF6XEIQA88NdN/PiFCJ8pGcXNl5yQ7nBEpAtT4hD+sGI7tz+1ioumDuXOT56ku7xFpFlKHBnulfUV3PTocs4YU8hP5p6qG/hEpEX6lshgK7bu5Z9+XcqE4r7ce1UJvXL1EEIRaZkSR4baWFHNNb9ayqCCPB74wgwG9Nb0rSLSOkocGWjX/hr+8b43cODBL8xgaP9e6Q5JRLoRJY4Ms+9QPVfd/yaV1XUsuOYMxhf3TXdIItLNKHFkkIZHiZRFq/jFP57OKSMHpjskEemG9MiRDBGLJ7jxkbdYurmSH805lXMnaTpdETk2uuLIAO7ON59YyXOrd/Hty6fyiWnD0x2SiHRjoSYOM5tpZmvNLGJm85tYP9rMlpjZW2a2wswuS1l3a7DdWjO7pLX7lCP98Nm1PFq6hXkfncjVZ+tRIiLSPqElDjPLBu4GLgWmAnPNbGqjat8CHnP3U0nOSf7TYNupwfKJwEzgp2aW3cp9SopfvbqRu5eUMXfGKL5+8fHpDkdEeoAwrzhmABF33+DudcBCYFajOg70D94PALYH72cBC9291t03ApFgf63ZpwT+uGIH//+p1Vw8dSjfmaVHiYhIxwgzcYwAtqQsbw3KUt0OfN7MtgJPAze2sG1r9gmAmV1vZqVmVhqNRo/1GLqtXftrmP+7FZw2eiA/1qNERKQDpfvbZC6wwN1HApcBvzazDonJ3e9x9xJ3LykuzqwRRA2d4XWxBP91xXQ9SkREOlSYw3G3AaNSlkcGZamuJdmHgbu/Zma9gKIWtm1pnxnvqRU7eP7dXXzzsimMKypIdzgi0sOEecWxFJhkZuPMLI9kZ/eiRnXeAy4EMLMpQC8gGtSbY2b5ZjYOmAS82cp9ZrTdVbXcvmgV00YN1GRMIhKK0K443D1mZvOAxUA2cL+7rzKzO4BSd18EfB2418xuItlRfrW7O7DKzB4DVgMx4J/dPQ7Q1D7DOobu6PanVnOgpp4ffPoUsrPUGS4iHc+S39M9W0lJiZeWlqY7jNA9t3oX1z1YytcuOp5/uXBSusMRkW7OzJa5e0nj8nR3jksH2Xeonm8+8Q6Tj+vHl86fkO5wRKQH07Oqeog7/7ia3dV13HfVGeRq6K2IhEjfMD3Ay+ujPFa6levPG8/JIwekOxwR6eGUOLq56toY83/7DuOLC/iK+jVEpBOoqaqb+88/rWH7vkP87z99WDf6iUin0BVHN/bmxkoeeG0zV314LCVjC9MdjohkCCWObqqmPs4tv13ByEG9ufmSE9IdjohkEDVVdVN3Pb+OjRXVPHTthyjI1z+jiHQeXXF0Q29v2cu9f9nAnDNGcc6konSHIyIZRomjm6mLJbjltyso7pfPv/7dlHSHIyIZSG0c3cxPX4ywZucBfnllCf175aY7HBHJQLri6EbW7NzP3UsizJo+nI9NHZrucEQkQylxdBOxeIJvPL6C/r1y+fbHT0x3OCKSwdRU1U3c98pGVmzdx0/mnkphQV66wxGRDKYrjm5gQ7SK/35uHRdPHcrlpwxLdzgikuGUOLq4RMK55bcryM/J4ruzT8JMkzOJSHqFmjjMbKaZrTWziJnNb2L9XWa2PHitM7O9QflHU8qXm1mNmc0O1i0ws40p66aHeQzp9tAbm1m6aQ/funwqQ/r3Snc4IiLh9XGYWTZwN3ARsBVYamaL3H11Qx13vyml/o3AqUH5EmB6UF4IRIBnU3Z/s7s/HlbsXcWWyoN875k1nDupiH84fWS6wxERAcK94pgBRNx9g7vXAQuBWc3Unws80kT5p4Fn3P1gCDF2WYmEM/93KzDgPz51spqoRKTLCDNxjAC2pCxvDcqOYGZjgHHAC02snsORCeVOM1sRNHXlH2Wf15tZqZmVRqPRtkefZve9spFXI7v51uVTGTmoT7rDERE5rKt0js8BHnf3eGqhmQ0DTgYWpxTfCkwGzgAKgVua2qG73+PuJe5eUlxcHE7UIVm9fT8/WLyWi6cOZc4Zo9IdjojIB4SZOLYBqd96I4OypjR1VQFwBfCEu9c3FLj7Dk+qBX5Fskmsx6ipj/OVhW8xsE8u3/v7U9REJSJdTpiJYykwyczGmVkeyeSwqHElM5sMDAJea2IfR/R7BFchWPIbdTawsoPjTqv/ePpd1pdX8cN/mKYb/USkSwptVJW7x8xsHslmpmzgfndfZWZ3AKXu3pBE5gAL3d1TtzezsSSvWF5qtOuHzawYMGA5cENYx9DZlqwt54HXNnPN2WM57/ju1bwmIpnDGn1f90glJSVeWlqa7jCaVVFVy8z/eZnBBXn8ft7Zmj9cRNLOzJa5e0njcj2rqgtwd+b/dgX7a+p56IszlDREpEvrKqOqMtpv3nyP598t55aZk5l8XP90hyMi0iwljjSLlFfxnT+s5txJRVxz1th0hyMi0iIljjSqiyX46qNv0Ts3mx/+wzSysjT0VkS6PvVxpNFdz69j5bb9/PzzpzNUDzAUkW5CVxxp8vqG3fz8pTLmnDGKmScdl+5wRERaTYkjDfYdqudrjy5nTGEf/u3yqekOR0SkTdRU1cncnW89uZJdB2r57ZfOoiBf/wQi0r3oiqOTPbl8G0+9vZ2vXjiJ6aMGpjscEZE2U+LoRFsqD3Lbk6soGTOIL390YrrDERE5JkocnSQWT3DTo8sBuOsz08nW0FsR6abUwN5JfvZiGaWb93DXZ6YxqlATM4lI96Urjk6wfMte/ufP6/n4tOHMnt7kJIgiIt2GEkfIqmtjfHXhWwztl893Z5+kiZlEpNtTU1XIvvOH1WyuPMgj153JgN656Q5HRKTddMURosWrdrJw6RZu+MgEzhw/ON3hiIh0iGYTh5kNMLPvmdkaM6s0s91m9m5Q1uJNCGY208zWmlnEzOY3sf4uM1sevNaZ2d6UdfGUdYtSyseZ2RvBPh8NpqXtkh58bRPjigq46WPHpzsUEZEO09IVx2PAHuB8dy9098HAR4Oyx5rb0MyygbuBS4GpwFwz+8DzNdz9Jnef7u7TgZ8Av0tZfahhnbt/IqX8+8Bd7j4xiOPaFo8yTSLlVZw6eiB5ObqwE5Geo6VvtLHu/n1339lQ4O473f37wJgWtp0BRNx9g7vXAQuBWc3Unws80twOLdmzfAHweFD0ADC7hTjS4kBNPbv21zJxSN90hyIi0qFaShybzewbZja0ocDMhprZLcCWFrYd0ajO1qDsCGY2BhgHvJBS3MvMSs3sdTNrSA6Dgb3uHmvFPq8Pti+NRqMthNrxyqLVAEwoVuIQkZ6lpcTxGZJf1i+Z2R4zqwReBAqBKzowjjnA4+4eTykbE0yS/lngf8xsQlt26O73uHuJu5cUFxd3YKitU1ZeBShxiEjP02zicPc9wK+AecCooJ9jirvfQrIpqjnbgFEpyyODsqbMoVEzlbtvC35uIJmsTgV2AwPNrGEYcXP7TKuyaBU5WcaYwbpLXER6lpZGVf0L8HuSiWOlmaX2Ufx7C/teCkwKRkHlkUwOixpXMrPJwCDgtZSyQWaWH7wvAs4GVru7A0uATwdVrwri63Ii5VWMGdyH3Gx1jItIz9LSDYDXAae7e5WZjQUeN7Ox7v4joNlboN09ZmbzgMVANnC/u68yszuAUndvSCJzgIVBUmgwBfiFmSVIJrfvufvqYN0twEIz+y7wFnBfaw+2M5VFq9RMJSI9UkuJI8vdqwDcfZOZnU8yeYyhhcQRbPM08HSjstsaLd/exHZ/BU4+yj430HIzWVrVxxNs3n2QS07UlLAi0vO01I6yy8ymNywESeRyoIijfLELbN59kFjCdcUhIj1SS4njSmBnaoG7x9z9SuC80KLq5sqiwYgq3cMhIj1Qs01V7r61mXWvdnw4PcPhxFFckOZIREQ6nob8hCBSXsXQ/vn066Wn4YpIz6PEEYKyaLX6N0Skx1Li6GDuzobyKj2jSkR6LCWODlZ+oJYDtTFdcYhIj6XE0cH0jCoR6emUODpYw4gqNVWJSE+lxNHBIuVVFORlM7R/frpDEREJhRJHByuLVjNhSF+Sc06JiPQ8ShwdrCxaxUT1b4hID6bE0YGqamPs2FejR42ISI+mxNGBNuhRIyKSAZQ4OpBGVIlIJgg1cZjZTDNba2YRM5vfxPq7zGx58FpnZnuD8ulm9pqZrTKzFWb2mZRtFpjZxpTtpjfeb7pEyqvIzjJGF+qKQ0R6rpYmcjpmZpYN3A1cBGwFlprZopSZ/HD3m1Lq30hyXnGAg8CV7r7ezIYDy8xssbvvDdbf7O6PhxX7sSorr2bM4D7k5ehCTkR6rjC/4WYAEXff4O51wEJgVjP15wKPALj7OndfH7zfDpQDxSHG2iE0XayIZIIwE8cIYEvK8tag7AjBVLTjgBeaWDcDyAPKUorvDJqw7jKzLnGnXSyeYNNuPRVXRHq+rtKmMgd43N3jqYVmNgz4NXCNuyeC4luBycAZQCFwS1M7NLPrzazUzEqj0Wh4kQfeqzxIfdzVMS4iPV6YiWMbMCpleWRQ1pQ5BM1UDcysP/BH4Jvu/npDubvv8KRa4Fckm8SO4O73uHuJu5cUF4ffylUWrQY0FFdEer4wE8dSYJKZjTOzPJLJYVHjSmY2GRgEvJZSlgc8ATzYuBM8uArBks/0mA2sDO0I2iBSrnnGRSQzhDaqyt1jZjYPWAxkA/e7+yozuwModfeGJDIHWOjunrL5FcB5wGAzuzoou9rdlwMPm1kxYMBy4IawjqEtyqJVDOmXT39NFysiPVxoiQPA3Z8Gnm5Udluj5dub2O4h4KGj7POCDgyxw2hElYhkiq7SOd6tuTuR8iomDFH/hoj0fEocHSBaVcuBmpieiisiGUGJowOUlQcjqtQxLiIZQImjA0SimmdcRDKHEkcHKCuvok9eNsMG9Ep3KCIioVPi6AANI6o0XayIZAIljg5QVl6lO8ZFJGMocbRTdW2M7ftq9IwqEckYShzttLGi4RlVShwikhmUONpJz6gSkUyjxNFOZdHkdLFjBvdJdygiIp1CiaOdyqJVjC7sQ35OdrpDERHpFEoc7RTRiCoRyTBKHO0QiyfYVHFQ/RsiklGUONph655D1MUTGlElIhlFiaMdGkZU6R4OEckkoSYOM5tpZmvNLGJm85tYf5eZLQ9e68xsb8q6q8xsffC6KqX8dDN7J9jnjy2Nz/koa3i4YZESh4hkjtBmADSzbOBu4CJgK7DUzBa5++qGOu5+U0r9G4FTg/eFwLeBEsCBZcG2e4CfAdcBb5CcXXAm8ExYx9GcsmgVRX3zGdBH08WKSOYI84pjBhBx9w3uXgcsBGY1U38u8Ejw/hLgOXevDJLFc8BMMxsG9Hf314M5yh8EZod3CM2LlFcxUbP+iUiGCTNxjAC2pCxvDcqOYGZjgHHACy1sOyJ435p9Xm9mpWZWGo1Gj+kAmuPulEWr1TEuIhmnq3SOzwEed/d4R+3Q3e9x9xJ3LykuLu6o3R62u7qOfYfqlThEJOOEmTi2AaNSlkcGZU2Zw/vNVM1tuy1435p9hkojqkQkU4WZOJYCk8xsnJnlkUwOixpXMrPJwCDgtZTixcDFZjbIzAYBFwOL3X0HsN/MzgxGU10J/D7EYziqwyOqlDhEJMOENqrK3WNmNo9kEsgG7nf3VWZ2B1Dq7g1JZA6wMOjsbti20sy+QzL5ANzh7pXB+y8DC4DeJEdTpWVEVaS8it652Qzrr+liRSSzhJY4ANz9aZJDZlPLbmu0fPtRtr0fuL+J8lLgpI6L8tiURauZMKSArCxNFysimaWrdI53O8npYtVMJSKZR4njGBysi7Ft7yElDhHJSEocx2BDNDldrEZUiUgmUuI4BodHVOmKQ0QykBLHMSgrryLLYGyRposVkcyjxHEMyqLVmi5WRDKWEscxKItqRJWIZC4ljjaKJ5wNFdW6Y1xEMpYSRxtt3XOQuliCibriEJEMpcTRRu8/o0rzcIhIZlLiaKOGp+Kqj0NEMpUSRxuVlVdT1DePgX3y0h2KiEhaKHG0UVm0ivG62hCRDKbE0QbuTiRapUeNiEhGU+Jog8rqOvYe1HSxIpLZlDjaoCx4uOGEYo2oEpHMFWriMLOZZrbWzCJmNv8oda4ws9VmtsrMfhOUfdTMlqe8asxsdrBugZltTFk3PcxjSKV5xkVEQpwB0MyygbuBi4CtwFIzW+Tuq1PqTAJuBc529z1mNgTA3ZcA04M6hUAEeDZl9ze7++NhxX40ZdEqeuVmMXxA787+aBGRLiPMK44ZQMTdN7h7HbAQmNWoznXA3e6+B8Ddy5vYz6eBZ9z9YIixtkpZtIrxRX01XayIZO4gWM8AAAlUSURBVLQwE8cIYEvK8tagLNXxwPFm9qqZvW5mM5vYzxzgkUZld5rZCjO7y8zym/pwM7vezErNrDQajR7rMXxApFwjqkRE0t05ngNMAs4H5gL3mtnAhpVmNgw4GVicss2twGTgDKAQuKWpHbv7Pe5e4u4lxcXF7Q70UF1c08WKiBBu4tgGjEpZHhmUpdoKLHL3enffCKwjmUgaXAE84e71DQXuvsOTaoFfkWwSC93Gimrc9YwqEZEwE8dSYJKZjTOzPJJNTosa1XmS5NUGZlZEsulqQ8r6uTRqpgquQjAzA2YDK8MIvrFIVCOqREQgxFFV7h4zs3kkm5mygfvdfZWZ3QGUuvuiYN3FZrYaiJMcLbUbwMzGkrxieanRrh82s2LAgOXADWEdQ6qy8irMYOxgXXGISGYLLXEAuPvTwNONym5Lee/A14JX4203cWRnOu5+QYcH2gpl0SpGDepDr1xNFysimS3dnePdhkZUiYgkKXG0QjzhbKyo1qNGRERQ4miV7XsPURtLaCiuiAhKHK2iZ1SJiLxPiaMVDs8zrisOEREljtYoi1ZRWJDHoAJNFysiosTRCpHyKibqakNEBFDiaJWyaLUeNSIiElDiaEFldR2V1XXq3xARCShxtOBwx7hGVImIAEocLSprGIqrKw4REUCJo0Vl0Sryc7IYMVDTxYqIgBJHiyLlVYwv1nSxIiINlDhaUBbVM6pERFIpcTSjpj7Olj0H9agREZEUShzNODxdrDrGRUQOCzVxmNlMM1trZhEzm3+UOleY2WozW2Vmv0kpj5vZ8uC1KKV8nJm9Eezz0WBa2lDoGVUiIkcKLXGYWTZwN3ApMBWYa2ZTG9WZBNwKnO3uJwJfTVl9yN2nB69PpJR/H7jL3ScCe4BrwzqGsvJqzGC8+jhERA4L84pjBhBx9w3uXgcsBGY1qnMdcLe77wFw9/LmdmhmBlwAPB4UPQDM7tCoU0SiVYwc1FvTxYqIpAgzcYwAtqQsb+XIOcSPB443s1fN7HUzm5myrpeZlQblDclhMLDX3WPN7BMAM7s+2L40Go0e0wFMGdaPy08Zfkzbioj0VDld4PMnAecDI4G/mNnJ7r4XGOPu28xsPPCCmb0D7Gvtjt39HuAegJKSEj+W4L58/sRj2UxEpEcL84pjGzAqZXlkUJZqK7DI3evdfSOwjmQiwd23BT83AC8CpwK7gYFmltPMPkVEJERhJo6lwKRgFFQeMAdY1KjOkySvNjCzIpJNVxvMbJCZ5aeUnw2sdncHlgCfDra/Cvh9iMcgIiKNhJY4gn6IecBi4F3gMXdfZWZ3mFnDKKnFwG4zW00yIdzs7ruBKUCpmb0dlH/P3VcH29wCfM3MIiT7PO4L6xhERORIlvxPfM9WUlLipaWl6Q5DRKRbMbNl7l7SuFx3jouISJsocYiISJsocYiISJsocYiISJtkROe4mUWBzce4eRFQ0YHhdDTF1z6Kr30UX/t09fjGuHtx48KMSBztYWalTY0q6CoUX/sovvZRfO3T1eM7GjVViYhImyhxiIhImyhxtOyedAfQAsXXPoqvfRRf+3T1+JqkPg4REWkTXXGIiEibKHGIiEibKHEEzGymma01s4iZzW9ifb6ZPRqsf8PMxnZibKPMbImZrTazVWb2lSbqnG9m+8xsefC6rbPiCz5/k5m9E3z2EU+UtKQfB+dvhZmd1omxnZByXpab2X4z+2qjOp16/szsfjMrN7OVKWWFZvacma0Pfg46yrZXBXXWm9lVnRjfD8xsTfDv94SZDTzKts3+LoQY3+1mti3l3/Cyo2zb7N96iPE9mhLbJjNbfpRtQz9/7ebuGf8CsoEyYDyQB7wNTG1U58vAz4P3c4BHOzG+YcBpwft+JCe8ahzf+cAf0ngONwFFzay/DHgGMOBM4I00/lvvJHljU9rOH3AecBqwMqXsP4H5wfv5wPeb2K4Q2BD8HBS8H9RJ8V0M5ATvv99UfK35XQgxvtuB/9eKf/9m/9bDiq/R+v8CbkvX+WvvS1ccSTOAiLtvcPc6YCEwq1GdWcADwfvHgQvNzDojOHff4e5/C94fIDm/SZNzrXdhs4AHPel1kjM5DktDHBcCZe5+rE8S6BDu/hegslFx6u/YA8DsJja9BHjO3SvdfQ/wHDCzM+Jz92c9Oc8OwOskZ+BMi6Ocv9Zozd96uzUXX/C9cQXwSEd/bmdR4kgaAWxJWd7KkV/Mh+sEfzz7SE4k1amCJrJTgTeaWP1hM3vbzJ4xsxM7NTBw4FkzW2Zm1zexvjXnuDPM4eh/sOk8fwBD3X1H8H4nMLSJOl3lPH6B5BVkU1r6XQjTvKAp7f6jNPV1hfN3LrDL3dcfZX06z1+rKHF0I2bWF/gt8FV3399o9d9INr9MA35CclreznSOu58GXAr8s5md18mf3yJLTmH8CeB/m1id7vP3AZ5ss+iSY+XN7JtADHj4KFXS9bvwM2ACMB3YQbI5qCuaS/NXG13+b0mJI2kbMCpleWRQ1mQdM8sBBgC7OyW65GfmkkwaD7v77xqvd/f97l4VvH8ayLXkfO2dwt23BT/LgSdINgmkas05DtulwN/cfVfjFek+f4FdDc13wc/yJuqk9Tya2dXA5cDnguR2hFb8LoTC3Xe5e9zdE8C9R/ncdJ+/HOBTwKNHq5Ou89cWShxJS4FJZjYu+F/pHGBRozqLgIYRLJ8GXjjaH05HC9pE7wPedff/Pkqd4xr6XMxsBsl/205JbGZWYGb9Gt6T7ERd2ajaIuDKYHTVmcC+lGaZznLU/+ml8/ylSP0duwr4fRN1FgMXm9mgoCnm4qAsdGY2E/gG8Al3P3iUOq35XQgrvtQ+s08e5XNb87cepo8Ba9x9a1Mr03n+2iTdvfNd5UVy1M86kiMuvhmU3UHyjwSgF8kmjgjwJjC+E2M7h2SzxQpgefC6DLgBuCGoMw9YRXKUyOvAWZ0Y3/jgc98OYmg4f6nxGXB3cH7fAUo6+d+3gGQiGJBSlrbzRzKB7QDqSbazX0uyz+zPwHrgeaAwqFsC/DJl2y8Ev4cR4JpOjC9Csn+g4XewYZThcODp5n4XOim+Xwe/WytIJoNhjeMLlo/4W++M+ILyBQ2/cyl1O/38tfelR46IiEibqKlKRETaRIlDRETaRIlDRETaRIlDRETaRIlDRETaRIlDRETaRIlDRETa5P8AGZO5qCxdG54AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rsgc93t3XPkN"
      },
      "source": [
        "Below is a variant of classifier using class weight and MLP layer but NOT lemmas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GugrUBWavbnB"
      },
      "source": [
        "classifier_mlp_weights = WSDClassifier(num_labels, device = 'cuda', use_mlp = True, hidden_size = 100,nbr_lemmas = len(lemma2i), lemma_embedding_size = 518, add_lemmas = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Tqpu49MvgrG",
        "outputId": "ecdc6a74-5dfe-4e05-d446-ce07d6546b26"
      },
      "source": [
        "# training\n",
        "from statistics import mean\n",
        "import numpy as np\n",
        "stop_early = 0\n",
        "early_stopping_patience = 6\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "LR = 0.0005\n",
        "n_epochs = 25\n",
        "import numpy as np\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "class_weight = compute_class_weight(\"balanced\", np.unique(list(label2i.values())),list(label2i.values()))\n",
        "\n",
        "class_weight = torch.tensor(class_weight, dtype=torch.float).to(classifier.device)\n",
        "\n",
        "#loss_function = nn.CrossEntropyLoss()\n",
        "loss_function = nn.NLLLoss(weight = class_weight) \n",
        "#optimizer = optim.SGD(classifier_mlp_weights.parameters(), lr=LR)\n",
        "optimizer = optim.Adam(classifier_mlp_weights.parameters(), lr=LR)\n",
        "\n",
        "config_name = 'sequoiaftb.asfalda_1_3.wsd.lr' + 'Adam' + str(LR) + '_bs' + str(BATCH_SIZE)\n",
        "out_model_file = './' + config_name + '.model'\n",
        "out_log_file = './' + config_name + '.log'\n",
        "\n",
        "\n",
        "# perte à chaque époque (sur le train / sur le validation set)\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "val_accs = []\n",
        "min_val_loss = None\n",
        "epoch_losses = []\n",
        "# pour tests: utiliser training sur dev\n",
        "#train_data = data['dev'] # data['train']\n",
        "train_data = wsd_data['train']\n",
        "val_data = wsd_data['val']\n",
        "dev_data = wsd_data['dev']\n",
        "classifier_mlp_weights.train()\n",
        "\n",
        "print('Training.....')\n",
        "acc = 0\n",
        "for epoch in range(n_epochs):  # loop over the dataset multiple times\n",
        "  classifier_mlp_weights.train()\n",
        "  \n",
        "  print('acc',acc)\n",
        "  print('Training..... epoch nr: ', epoch)\n",
        "  for b_tid_seqs, b_tg_trks, b_labels, b_lemma_idx in train_data.make_batches(64, shuffle_data=True):\n",
        "    \n",
        "  \n",
        "    optimizer.zero_grad()\n",
        "    b_tid_seqs = torch.tensor(b_tid_seqs, device=classifier_mlp_weights.device).to(classifier_mlp_weights.device)\n",
        "    b_tg_trks = torch.tensor(b_tg_trks, device=classifier_mlp_weights.device).to(classifier_mlp_weights.device)\n",
        "    b_labels = torch.tensor(b_labels, device=classifier_mlp_weights.device).to(classifier_mlp_weights.device)\n",
        "    b_lemma_idx = torch.tensor(b_lemma_idx, device=classifier_mlp_weights.device).to(classifier_mlp_weights.device)\n",
        "    log_probs = classifier_mlp_weights(b_tid_seqs, b_tg_trks, b_lemma_idx).to(classifier_mlp_weights.device)\n",
        "  \n",
        "    loss = loss_function(log_probs,  b_labels)\n",
        "    \n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_losses.append(loss.item())\n",
        "  epoch_losses.append(sum(train_losses)/len(train_losses))\n",
        "  \n",
        "  pred_labels, val_losses, val_acc, _, _ =classifier_mlp_weights.run_on_dataset(val_data, optimizer, batch_size=32, validation_use=True)\n",
        "  val_accs.append(val_acc)\n",
        "  print('--------')\n",
        "  print('train loss: ',epoch_losses[-1],'val accuracy: ', val_acc)\n",
        "  print('--------')\n",
        " \n",
        "  if val_acc>=acc:\n",
        "    acc=val_acc\n",
        "  else:\n",
        "    early_stopping_patience +=1\n",
        "    if early_stopping_patience == stop_early:\n",
        "      print('Stopping early...')\n",
        "      break\n",
        "\n",
        "  \n",
        "    \n",
        "  \n",
        " \n",
        "\n",
        "\n",
        "        \n",
        "  \n",
        "           \n",
        "# TODO HERE\n",
        "# training\n",
        "# - basic : train for NB_EPOCHS +\n",
        "# - BONUS : early stopping: stop epoch loop as soon as accuracy on dev decreases+\n",
        "\n",
        "# don't forget to toggle \n",
        "# - classifier.train() when training on train\n",
        "# - classifier.eval() when evaluating on val corpus\n",
        "\n",
        "\n",
        "print(\"train losses: %s\" % ' / '.join([ \"%.4f\" % x for x in epoch_losses]))\n",
        "print(\"val   losses: %s\" % ' / '.join([ \"%.4f\" % x for x in val_losses]))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training.....\n",
            "acc 0\n",
            "Training..... epoch nr:  0\n",
            "--------\n",
            "train loss:  2.4700915464430255 val accuracy:  0.6634887005649718\n",
            "--------\n",
            "acc 0.6634887005649718\n",
            "Training..... epoch nr:  1\n",
            "--------\n",
            "train loss:  1.9382467915803308 val accuracy:  0.7461158192090396\n",
            "--------\n",
            "acc 0.7461158192090396\n",
            "Training..... epoch nr:  2\n",
            "--------\n",
            "train loss:  1.6437570344661427 val accuracy:  0.781367702448211\n",
            "--------\n",
            "acc 0.781367702448211\n",
            "Training..... epoch nr:  3\n",
            "--------\n",
            "train loss:  1.4556877709670213 val accuracy:  0.7986111111111112\n",
            "--------\n",
            "acc 0.7986111111111112\n",
            "Training..... epoch nr:  4\n",
            "--------\n",
            "train loss:  1.3212100524866082 val accuracy:  0.7996704331450094\n",
            "--------\n",
            "acc 0.7996704331450094\n",
            "Training..... epoch nr:  5\n",
            "--------\n",
            "train loss:  1.2209217516075068 val accuracy:  0.821680790960452\n",
            "--------\n",
            "acc 0.821680790960452\n",
            "Training..... epoch nr:  6\n",
            "--------\n",
            "train loss:  1.141903290681901 val accuracy:  0.8264477401129944\n",
            "--------\n",
            "acc 0.8264477401129944\n",
            "Training..... epoch nr:  7\n",
            "--------\n",
            "train loss:  1.07873979178275 val accuracy:  0.8277424670433146\n",
            "--------\n",
            "acc 0.8277424670433146\n",
            "Training..... epoch nr:  8\n",
            "--------\n",
            "train loss:  1.025772965881056 val accuracy:  0.8282721280602636\n",
            "--------\n",
            "acc 0.8282721280602636\n",
            "Training..... epoch nr:  9\n",
            "--------\n",
            "train loss:  0.9805041235209417 val accuracy:  0.837511770244821\n",
            "--------\n",
            "acc 0.837511770244821\n",
            "Training..... epoch nr:  10\n",
            "--------\n",
            "train loss:  0.942586006578013 val accuracy:  0.842337570621469\n",
            "--------\n",
            "acc 0.842337570621469\n",
            "Training..... epoch nr:  11\n",
            "--------\n",
            "train loss:  0.9097324055259521 val accuracy:  0.8335687382297552\n",
            "--------\n",
            "acc 0.842337570621469\n",
            "Training..... epoch nr:  12\n",
            "--------\n",
            "train loss:  0.8808361027116279 val accuracy:  0.8348634651600754\n",
            "--------\n",
            "acc 0.842337570621469\n",
            "Training..... epoch nr:  13\n",
            "--------\n",
            "train loss:  0.8549144969669326 val accuracy:  0.8415136534839924\n",
            "--------\n",
            "acc 0.842337570621469\n",
            "Training..... epoch nr:  14\n",
            "--------\n",
            "train loss:  0.8317189445022697 val accuracy:  0.84310263653484\n",
            "--------\n",
            "acc 0.84310263653484\n",
            "Training..... epoch nr:  15\n",
            "--------\n",
            "train loss:  0.8112850315026061 val accuracy:  0.8415136534839924\n",
            "--------\n",
            "acc 0.84310263653484\n",
            "Training..... epoch nr:  16\n",
            "--------\n",
            "train loss:  0.7925157343493402 val accuracy:  0.8409251412429378\n",
            "--------\n",
            "acc 0.84310263653484\n",
            "Training..... epoch nr:  17\n",
            "--------\n",
            "train loss:  0.7749914812655151 val accuracy:  0.8481638418079096\n",
            "--------\n",
            "acc 0.8481638418079096\n",
            "Training..... epoch nr:  18\n",
            "--------\n",
            "train loss:  0.759392044372217 val accuracy:  0.8449858757062146\n",
            "--------\n",
            "acc 0.8481638418079096\n",
            "Training..... epoch nr:  19\n",
            "--------\n",
            "train loss:  0.7452590936263704 val accuracy:  0.8409839924670434\n",
            "--------\n",
            "acc 0.8481638418079096\n",
            "Training..... epoch nr:  20\n",
            "--------\n",
            "train loss:  0.7320001420552977 val accuracy:  0.846810263653484\n",
            "--------\n",
            "acc 0.8481638418079096\n",
            "Training..... epoch nr:  21\n",
            "--------\n",
            "train loss:  0.7193947119198577 val accuracy:  0.8478695856873822\n",
            "--------\n",
            "acc 0.8481638418079096\n",
            "Training..... epoch nr:  22\n",
            "--------\n",
            "train loss:  0.7078353842441101 val accuracy:  0.852930790960452\n",
            "--------\n",
            "acc 0.852930790960452\n",
            "Training..... epoch nr:  23\n",
            "--------\n",
            "train loss:  0.6965644932688416 val accuracy:  0.8516360640301318\n",
            "--------\n",
            "acc 0.852930790960452\n",
            "Training..... epoch nr:  24\n",
            "--------\n",
            "train loss:  0.6864654910201808 val accuracy:  0.8526953860640302\n",
            "--------\n",
            "train losses: 2.4701 / 1.9382 / 1.6438 / 1.4557 / 1.3212 / 1.2209 / 1.1419 / 1.0787 / 1.0258 / 0.9805 / 0.9426 / 0.9097 / 0.8808 / 0.8549 / 0.8317 / 0.8113 / 0.7925 / 0.7750 / 0.7594 / 0.7453 / 0.7320 / 0.7194 / 0.7078 / 0.6966 / 0.6865\n",
            "val   losses: 0.4936 / 0.4713 / 0.4274 / 0.3889 / 0.3491 / 0.6803 / 0.4361 / 0.7116 / 0.2688 / 0.2641 / 0.6634 / 0.1300 / 0.3933 / 0.2429 / 0.1665 / 0.4976 / 0.2892 / 0.8308 / 0.3074 / 0.7305 / 0.3445 / 0.5727 / 0.2111 / 0.2811 / 0.1458 / 0.3656 / 0.2555 / 0.2266 / 0.4744 / 0.5526 / 0.7961 / 0.4539 / 0.3430 / 0.6532 / 0.5964 / 0.2374 / 0.5926 / 0.3336 / 0.4169 / 0.2277 / 0.6507 / 0.4832 / 0.3359 / 0.6586 / 0.3503 / 0.7294 / 0.1902 / 0.3125 / 0.7607 / 0.8454 / 0.6253 / 0.4762 / 0.4385 / 0.4368 / 0.3357 / 1.1616 / 0.5456 / 0.7593 / 0.9372\n",
            "mean train loss : \n",
            "mean val loss : \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "iICXMb1IwDbZ",
        "outputId": "8b1373c2-0905-4dfe-c329-0f6238b0ecda"
      },
      "source": [
        "import matplotlib.pyplot as plt 0.8524011299435028\n",
        "plt.plot(epoch_losses)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD4CAYAAAD2FnFTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8dcnO9nICoEsBBJUcGGLKMQFx9atWmtrHa1jrW2H2tGO7Tiddjrza2fazkw70zpTra3SapcZt9ZtnKpV646gEhBEQNmXsAYSwhICWT6/P+4JXsINa25OyH0/H4/7yL3f7/fkfji54Z1zvmcxd0dERKS7pLALEBGR/kkBISIiMSkgREQkJgWEiIjEpIAQEZGYUsIuoDcVFRV5ZWVl2GWIiJww5s6du9Xdi2P1DaiAqKyspK6uLuwyREROGGa2pqe+uO1iMrNyM3vZzBab2SIzuy3GmGlm1mxm84PHt6P6LjGzD8xsuZl9M151iohIbPHcgmgHbnf3eWaWA8w1sxfcfXG3ca+7++XRDWaWDNwNfBSoB+aY2VMxlhURkTiJ2xaEu29093nB853AEqD0CBefDCx395Xuvg94GLgyPpWKiEgsfXIUk5lVAhOAt2J0TzGzBWb2rJmdGrSVAuuixtTTQ7iY2XQzqzOzuoaGhl6sWkQkscU9IMwsG3gM+Kq77+jWPQ8Y4e7jgLuAJ4/2+7v7DHevcfea4uKYE/EiInIM4hoQZpZKJBwecPfHu/e7+w533xU8fwZINbMiYD1QHjW0LGgTEZE+Es+jmAy4D1ji7nf0MKYkGIeZTQ7q2QbMAUab2UgzSwOuBZ6KV60iInKweB7FVAvcACw0s/lB27eACgB3vwe4GviymbUDe4BrPXL98XYzuxV4DkgG7nf3RfEocm97B796YzWnDR/MOaOL4vEWIiInpLgFhLvPBOwwY34K/LSHvmeAZ+JQ2gFSk5L4xWsrOf+kYgWEiEiUhL8WU1KSMaWqkDdWbEU3TxIR+VDCBwRAbXURm3fsZUXD7rBLERHpNxQQQG1VZNfSrBVbQ65ERKT/UEAAFYWZlOUP4o3lCggRkS4KiEBtVRGzV2yjo1PzECIioIDYb2p1ITta21m0oTnsUkRE+gUFRGBqMA/xxvJtIVciItI/KCACxTnpnDw0RxPVIiIBBUSUqdWFzFndyN72jrBLEREJnQIiSm1VEa1tncxbsz3sUkREQqeAiDJ5VAFJpvMhRERAAXGA3IxUzijL0/kQIiIoIA5SW13Igvpmdra2hV2KiEioFBDd1FYV0dHpvL2qMexSRERCpYDoZuKIfNJTknQ+hIgkPAVENxmpydRU5muiWkQSngIihqlVRby/aSdbd+0NuxQRkdAoIGKore66/Ld2M4lI4lJAxHB66WByMlKYpcNdRSSBKSBiSE4yzh5VqC0IEUlocQsIMys3s5fNbLGZLTKz22KMud7M3jWzhWY2y8zGRfWtDtrnm1ldvOrsSW1VIWsbW1jX2NLXby0i0i/EcwuiHbjd3ccCZwO3mNnYbmNWAee7++nA94AZ3fovcPfx7l4Txzpj+nAeQruZRCQxxS0g3H2ju88Lnu8ElgCl3cbMcvem4OWbQFm86jla1UOyGZKTrvMhRCRh9ckchJlVAhOAtw4x7AvAs1GvHXjezOaa2fRDfO/pZlZnZnUNDQ29UW7X92VqVWQewl23IRWRxBP3gDCzbOAx4KvuvqOHMRcQCYhvRDWf4+4TgUuJ7J46L9ay7j7D3Wvcvaa4uLhXa59aXcTWXXtZunlXr35fEZETQVwDwsxSiYTDA+7+eA9jzgB+CVzp7vv357j7+uDrFuAJYHI8a42lax5CV3cVkUQUz6OYDLgPWOLud/QwpgJ4HLjB3ZdGtWeZWU7Xc+Ai4L141dqT0rxBVBZmaqJaRBJSShy/dy1wA7DQzOYHbd8CKgDc/R7g20Ah8LNIntAeHLE0FHgiaEsBHnT3P8ax1h5NqSriDws20N7RSUqyThsRkcQRt4Bw95mAHWbMF4EvxmhfCYw7eIm+V1tdyENvr+Xd9c1MrMgPuxwRkT6jP4kPY8qoQgBddkNEEo4C4jAKs9MZMyxX50OISMJRQByB2qpC5q5torWtI+xSRET6jALiCNRWF7GvvZO61U2HHywiMkAoII7A5JEFpCQZb+hwVxFJIAqII5CVnsL48jxNVItIQlFAHKGp1UUsXN9M8562sEsREekTCogjVFtVSKfDmyt1NJOIJAYFxBGaUJHPoNRk7WYSkYShgDhCaSlJnDmygDd0G1IRSRAKiKNQW1XI8i272LyjNexSRETiTgFxFLou/z1bWxEikgAUEEdh7LBc8jJTdX8IEUkICoijkJRkTBml25CKSGJQQBylqVWFrN++hzXbWsIuRUQkrhQQR2lq121IddkNERngFBBHaVRRFiW5GczS5b9FZIBTQBwlM2NqdSGzVmyls1PzECIycCkgjkFtVRFNLW0s2bQj7FJEROJGAXEMus6H0G4mERnI4hYQZlZuZi+b2WIzW2Rmt8UYY2Z2p5ktN7N3zWxiVN+NZrYseNwYrzqPRcngDEYVZ2miWkQGtHhuQbQDt7v7WOBs4BYzG9ttzKXA6OAxHfg5gJkVAN8BzgImA98xs/w41nrUaquKeHtVI/vaO8MuRUQkLuIWEO6+0d3nBc93AkuA0m7DrgR+6xFvAnlmNgy4GHjB3RvdvQl4AbgkXrUei9rqQlr2dbCgfnvYpYiIxEWfzEGYWSUwAXirW1cpsC7qdX3Q1lN7rO893czqzKyuoaGht0o+rLNHFWKGLrshIgNW3APCzLKBx4CvunuvH/bj7jPcvcbda4qLi3v72/coLzON04YP1kS1iAxYcQ0IM0slEg4PuPvjMYasB8qjXpcFbT219ytTqwt5Z10TLfvawy5FRKTXxfMoJgPuA5a4+x09DHsK+GxwNNPZQLO7bwSeAy4ys/xgcvqioK1fqa0qoq3DeXtVY9iliIj0upQ4fu9a4AZgoZnND9q+BVQAuPs9wDPAZcByoAW4KehrNLPvAXOC5b7r7v3uf+EzKwtIS05i1optTDt5SNjliIj0qrgFhLvPBOwwYxy4pYe++4H741BarxmUlsyEijxNVIvIgKQzqY9TbXURizfuoGn3vrBLERHpVQqI41RbXYg7zF6po5lEZGBRQBynM8ryyElP4dn3NoVdiohIr1JAHKfU5CSuO6uCp9/dwOqtu8MuR0Sk1yggesEXzx1JSnISP39lRdiliIj0GgVELxiSk8G1Z5bz+Dv1bNi+J+xyRER6hQKil3zp/CrcYcZrK8MuRUSkVyggeklp3iCumlDKQ2+vpWHn3rDLERE5bgqIXvTlaVW0dXRy38xVYZciInLcFBC9aFRxNh87Yzj/PXs121t04pyInNgUEL3slguq2L2vg1/PWh12KSIix0UB0ctOKcnlI2OG8qs3VrNrry4DLiInLgVEHNz6Z9U072njgTfXhF2KiMgxU0DEwfjyPM6pLuIXr6+ita0j7HJERI6JAiJObv2zarbu2ssjc9YdfrCISD+kgIiTs0YWUDMin3tfXcG+9s6wyxEROWoKiDgxM275s2o2NLfy5Dv97nbaIiKHpYCIo2knFXNaaS4/f3UFHZ0edjkiIkdFARFHZsYt06pZtXU3Ty/cGHY5IiJHJW4BYWb3m9kWM3uvh/6vm9n84PGemXWYWUHQt9rMFgZ9dfGqsS9cfGoJ1UOyuful5XRqK0JETiDx3IL4NXBJT53u/h/uPt7dxwN/D7zq7o1RQy4I+mviWGPcJSUZt1xQxQebd/KnJZvDLkdE5IjFLSDc/TWg8bADI64DHopXLWG74ozhVBRkcvfLy3HXVoSInBhCn4Mws0wiWxqPRTU78LyZzTWz6eFU1ntSkpO4+fwqFtQ3M3P51rDLERE5IqEHBHAF8Ea33UvnuPtE4FLgFjM7r6eFzWy6mdWZWV1DQ0O8az1mn5pUSkluBj99aXnYpYiIHJH+EBDX0m33kruvD75uAZ4AJve0sLvPcPcad68pLi6Oa6HHIz0lmennjeKtVY3MWX2ke95ERMITakCY2WDgfOB/o9qyzCyn6zlwERDzSKgTzXWTKyjMStNWhIicEOJ5mOtDwGzgZDOrN7MvmNnNZnZz1LCrgOfdfXdU21BgppktAN4Gnnb3P8arzr40KC2Zz58zkleXNrCwvjnsckREDskG0lE1NTU1XlfXv0+b2NHaRu0PXqK2qoh7bpgUdjkikuDMbG5PpxP0hzmIhJKbkcpNUyv546JNLN28M+xyRER6pIAIwU21I8lMS+ZnL2suQkT6LwVECPKz0rj+rAqeWrCBNdt2H34BEZEQKCBC8pfnjiIlOYl7Xl0RdikiIjEpIEIyJDeDa2rKeHRuPRub94RdjojIQRQQIfrSeVW4w4zXVoZdiojIQRQQISovyOQTE0p56O21bN21N+xyREQOoIAI2ZenVbG3vZM7X1wWdikiIgdQQISsqjibG6dU8tvZa3hhse4XISL9xyEDwswGm9kPzOx9M2s0s21mtiRoy+urIge6v7/sFE4rzeVvf7+A9ds1YS0i/cPhtiB+BzQB09y9wN0LgQuCtt/Fu7hEkZ6SzE+vm0hHp/OVB+fR1tEZdkkiIocNiEp3/6G7b+pqcPdN7v5DYER8S0sslUVZ/NsnT2fe2u38+PmlYZcjInLYgFhjZn9nZkO7GsxsqJl9A1gX39ISzxXjhvOZsyq459UVvPzBlrDLEZEEd7iA+HOgEHjVzJrMrBF4BSgArolzbQnp25eP5ZSSHG7/3QI2NbeGXY6IJLBDBoS7NwG/Am4FyoN5iDHu/g0OcZc3OXYZqcncff1EWts6+OuH36Fd8xEiEpLDHcX010Tu9nYr8J6ZXRnV/a/xLCyRVRVn8y9Xncbbqxp1foSIhCblMP1/CUxy911mVgk8amaV7v4TwOJdXCK7akIZs5Zv466XlzN5ZCHnjC4KuyQRSTCHm4NIcvddAO6+GpgGXGpmd6CAiLt/vvJUqouz+eoj89myU/MRItK3DhcQm81sfNeLICwuB4qA0+NZmEBmWgp3Xz+RXXvb+OrD8+noHDi3hxWR/u9wAfFZYFN0g7u3u/tngfPiVpXsd9LQHL778dOYtWIbd+sOdCLShw53FFN99Ely3freONSyZna/mW0xs/d66J9mZs1mNj94fDuq7xIz+8DMlpvZN4/kHzKQfbqmjE+MH85//Wkpb67cFnY5IpIg4nmxvl8DlxxmzOvuPj54fBfAzJKBu4FLgbHAdWY2No519ntmxvevOp3Kwixue/gdtunS4CLSB+IWEO7+GtB4DItOBpa7+0p33wc8DFx5mGUGvOz0FH76mYk0tbTxN79bQKfmI0QkzsK+3PcUM1tgZs+a2alBWykHXsajPmiLycymm1mdmdU1NDTEs9bQjR2ey7cvH8urSxu4V3ehE5E4CzMg5gEj3H0ccBfw5LF8E3ef4e417l5TXFzcqwX2R9efVcHHzhjGj57/gLlrjmUDTUTkyIQWEO6+I+oci2eAVDMrAtYD5VFDy4I2ITIf8W+fPJ3SvEF85cF3aNq9L+ySRGSACi0gzKzEzCx4PjmoZRswBxhtZiPNLA24FngqrDr7o9yMVO7+zEQadu3l648uwF3zESLS++IWEGb2EDAbONnM6s3sC2Z2s5ndHAy5msj1nRYAdwLXekQ7kWs/PQcsAX7n7oviVeeJ6vSywXzrsjH8ackW7pu5KuxyRGQAsoH012dNTY3X1dWFXUafcXdu/p+5vPT+Fh750hQmVuSHXZKInGDMbK6718TqC/soJjkOZsa/f2ocJYMz+Nz9b/PO2qawSxKRAUQBcYIbnJnKg188m7zMNP7il28xe4XOtBaR3qGAGADKCzL5/c1TGJ43iM/96m3drlREeoUCYoAYmpvBI1+aQvWQbKb/to5nF24MuyQROcEpIAaQgqw0HvzLszmjLI9bHpzHY3Prwy5JRE5gCogBZvCgVH77+cmcPaqQ23+/gP9+c03YJYnICUoBMQBlpadw/+fO5MJThvD/nnyPe19dEXZJInICUkAMUBmpydxzwyQ+dsYw/u3Z97njhaU641pEjkpK2AVI/KQmJ3HntRPITE3mzheXsXtvO//4sTEEVzgRETkkBcQAl5xk/PBTZ5CVnsJ9M1fRsq+D73/iNJKTFBIicmgKiASQlGR854qxZKUnc/fLK9izr50ffXocKcnawygiPVNAJAgz4+sXn0JmWgr/8dwHtOzr4K7PTCA9JTns0kSkn9KfkAnmlguq+acrxvL84s188Td17NnXEXZJItJPKSAS0OdqR/LvnzqDN5Zv5cb732Zna1vYJYlIP6SASFDXnFnOT66dwLy1TVz/y7d0ZzoROYgCIoFdMW449/zFJN7ftJPL75pJ3Wrd41pEPqSASHAfGTuUR6afTXKScc29s7njhaW0d3SGXZaI9AMKCGFCRT5P//U5fGJCKXe+uIxr7p3N2m0tYZclIiFTQAgAORmp3HHNeO68bgLLtuzisjtf54l3dDVYkUSmgJADfHzccJ697VzGDMvha48s4LaH32GHjnISSUhxCwgzu9/MtpjZez30X29m75rZQjObZWbjovpWB+3zzawuXjVKbGX5mTw8fQq3f/Qk/vDuRi79r9c1gS2SgOK5BfFr4JJD9K8Cznf304HvATO69V/g7uPdvSZO9ckhJCcZX7lwNL+/eYomsEUSVNwCwt1fA3r8s9PdZ7l7U/DyTaAsXrXIsZsYTGBfNaGMO19cxqc1gS2SMPrLHMQXgGejXjvwvJnNNbPph1rQzKabWZ2Z1TU0NMS1yESVk5HKj68Zx13XTWB5MIH9+Lx63V9CZIALPSDM7AIiAfGNqOZz3H0icClwi5md19Py7j7D3Wvcvaa4uDjO1Sa2K4IJ7LHDcvmb3y3gtofnawJbZAALNSDM7Azgl8CV7r6tq93d1wdftwBPAJPDqVC6K8vP5KHpZ3P7R0/i6YWRCew5msAWGZBCCwgzqwAeB25w96VR7VlmltP1HLgIiHkklISjawL70WAC+8/vnc2/PrNEF/0TGWAsXvuRzewhYBpQBGwGvgOkArj7PWb2S+BTwJpgkXZ3rzGzUUS2GiByv4oH3f1fjuQ9a2pqvK5OR8X2pV172/ne/y3mkbp1FGWn8/WLT+LqSeW6Y53ICcLM5vZ0tGjcAiIMCojwLFi3nX/+v0XMW7ud00pz+fblpzJ5ZEHYZYnIYRwqIEKfpJaBYVx5Ho99eSo/uXY823bt45p7Z3PLg/Oob9IhsSInKgWE9Boz48rxpbx0+zRuu3A0Ly7ZzIU/fpUfP/8BLfvawy5PRI6SAkJ63aC0ZL720ZN46fZpXHxqCXe9tJwLfvQKT7xTT2fnwNmlKTLQKSAkbobnDeLO6ybw2JenMDQ3g689soBP/nwW76xtOvzCIhI6BYTE3aQRBTz5V7X86NPj2LB9D1f9bBZfe2Q+m5pbwy5NRA5BASF9IinJuHpSGS//7TRuuaCKpxdu5IIfvcJdLy6jta0j7PJEJAYFhPSprPQUvn7xKbz4N+cz7eRifvzCUi788as8Nreefe26UqxIf6LzICRUs1ds4/tPL2bRhh2U5Gbw+XMquW5yBTkZqWGXJpIQdKKc9GvuzitLG5jx6kpmr9xGTnoKnzmrgptqR1IyOCPs8kQGNAWEnDAW1jcz4/WVPP3uBpKTjI+PK2X6eaM4uSQn7NJEBiQFhJxw1jW2cN/MVTwyZx172jo4/6RivnTeKKZUFWKm6zyJ9BYFhJywtrfs43/eXMOvZ61m6659nFaay/TzqrjstBJSknWMhcjxUkDICa+1rYMn3lnPL15bycqtuynLH8QXzhnJNTXlZKWnhF2eyAlLASEDRmen86clm5nx2krq1jQxeFAqN5w9ghumjGBoria0RY6WAkIGpLlrmpjx2gqeX7wZA847qZirJ5XxkTFDyUhNDrs8kROCAkIGtNVbd/P7uet4fN56Nja3kpuRwhXjhnP1pDLGl+dpUlvkEBQQkhA6Op3ZK7bx6Nx1/HHRJlrbOhlVnMXVk8r45IQynVMhEoMCQhLOztY2nlm4kUfn1jNndRNJBrXVRVw9qYyLTy3RLiiRgAJCEtrqrbt5fF49j81bz/rte8hJT+HyccO4elIZEyvytQtKEpoCQoTIEVBvrtrGo3PreXbhJva0dTCyKItPTSzlyvGllBdkhl2iSJ8LLSDM7H7gcmCLu58Wo9+AnwCXAS3A59x9XtB3I/CPwdDvu/tvDvd+Cgg5Urv2tvNssAvqrVWNAJxRNphLTxvGZaeXMKIwK+QKRfpGmAFxHrAL+G0PAXEZ8BUiAXEW8BN3P8vMCoA6oAZwYC4wyd0PeSsyBYQci3WNLTy9cCPPLtzIgvpmAE4dnstlpw/j0tNKGFWcHXKFIvET6i4mM6sE/tBDQNwLvOLuDwWvPwCmdT3c/UuxxvVEASHHa11jC88t2sQzCzcyb+12AE4pydm/ZTF6qC4aKAPLoQIi7GsUlALrol7XB209tR/EzKYD0wEqKiriU6UkjPKCTL547ii+eO4oNjbv4Y/vbeLZhZv4rxeX8p9/Wkr1kGwuO62ES08fxiklOZrglgEt7IA4bu4+A5gBkS2IkMuRAWTY4EHcVDuSm2pHsmVHK38Mtix++vJy7nxpOSOLsrj0tBIuO30Ypw7PVVjIgBN2QKwHyqNelwVt64nsZopuf6XPqhLpZkhuBp+dUslnp1TSsHMvzy+ObFnc+9pKfvbKCoYNzuCc6iLOGV1EbXURRdnpYZcsctzCnoP4GHArH05S3+nuk4NJ6rnAxGDoPCKT1I2Hei/NQUhfa9y9jxcWb+LVpQ28sXwbzXvaABgzLJdzg7CYXFnAoDSdmCf9U5hHMT1EZEugCNgMfAdIBXD3e4LDXH8KXELkMNeb3L0uWPbzwLeCb/Uv7v6rw72fAkLC1NHpvLe+mZnLtzJz2VbmrmliX0cnaclJ1FTmU1tdxLmjizh1+GCSk7Q7SvoHnSgnEoKWfe3MWd3EzGUNvL5sK+9v2glAXmYqU6sKOae6mHNHF+kEPQlVfz6KSWTAykxL4fyTijn/pGIAGnbuZdaKrby+LLKF8czCTQBUFGQytaqQmsoCzqzMp6IgUxPe0i9oC0IkBO7OiobdzFzWwMzlW3l7VSM7WtsBKM5J58zKfGpGFHBmZQFjhuXo9qoSN9qCEOlnzIzqIdlUD8nmc7Uj6ex0lm3ZxZzVjcxd08Sc1Y37tzAy05KZWJFPTWU+Z1YWML48T7dZlT6hLQiRfmpj8x7qVjdRt7qROaubWLJpB+6QnGScOjw32MLIZ1JlPkNydK8LOTaapBYZAHa0tvHO2u1BYDTyztrt7G3vBKC8YBDjy/MZVzaYCRV5nDp8sO55IUdEu5hEBoDcjNQDJr33tXeyaEPz/rCYu7qR/1uwAYCUJOPkkhzGl+cxrjyP8eV5VBVn6/BaOSoKCJETVFpKEhMq8plQkb+/bcuOVuav286C+u0sWNfMU/M38MBbawHITk/h9NLBjK/IY1xZJDR0G1Y5FO1iEhnAOjudlVt3R0IjCI7FG3bQ3hn5vS/JzWBc+WDOKMtjzLAcxgzLpSQ3Q4fZJhDtYhJJUElJHx4tdfWkMgBa2zpYvHEHC9ZtZ37weG7R5v3L5GWmMqYklzHDcveHxuih2aSnaE4j0SggRBJMRmrksNmJUbumdrS28cGmnSzZuIMlG3eweONOHnx7Da1tkUnwlCSjqjh7f2B0PYpzdFHCgUwBISLkZqRyZmXkxLwuHZ3O6m27I4GxIRIcb65s5Mn5G/aPKcpOZ8ywHE4pyWH00BxGD8lm9NAcsnWexoCgn6KIxJQcbDVUFWdz+RnD97c37d4XbGXsYMnGyFbHb2avYV9wyC3A8MEZ+wPjpKE5VA/NZvSQbHIyUsP4p8gxUkCIyFHJz0pjanURU6uL9re1d3SyrmkPyzbvZNmWXSzbvJOlm3fx5spt+8/VABh2QHBkUz0kh9FDs8lVcPRLCggROW4pyUmMLMpiZFEWF536YXtHp1Pf1MLSzbtYtmUny4KvD7y1bf/8BsCQnHRGFWcxqjibUUVZVBVnM6o4i9K8QboOVYgUECISN8lJxojCLEYUZvHRsUP3t3d0Ouub9rBsS2RLY0XDLlY27OKZhRvZ3tK2f1xachIVhZmMKgrCozhr//OCrLQw/kkJRQEhIn0uOcmoKMykojCTC8cMPaCvcfc+VjbsYmXDblZu3R15vnU3L3+whbaOD8/bystMZVRRFiOLIsFRUZDJiMJMKgoyyctUePQGBYSI9CsFWWkUZBVQE3VEFUTmOeqb9rBq6+7IFkcQHq8va+CxefUHjM3NSGFEYVYkhAoyGVGQuf/5sMGDdMmRI6SAEJETQkpyEpVFWVQWZXHBKUMO6GvZ187axhbWbGth7baWyPPGFhatb+a59zbtP3McIrutyvIH7Q+MrkdZfiZlBYM0YR5FASEiJ7zMtBROKcnllJLcg/raOzrZ2Nz6YYA0trC2cTdrtrUwd3UTO/e2HzA+NyMlEhb5gygviHztel2WPyihDtVVQIjIgJaSnER5QSblBZnUVh/Y5+5sb2ljbWML67fvob6phfqmPdQ37WH1tt28vmwre9o6Dlhm8KDU/WHRFRyleYMYnjeIYYMzKMhKGzDXsoprQJjZJcBPgGTgl+7+g279/wlcELzMBIa4e17Q1wEsDPrWuvvH41mriCQeMyM/K438rDTGlecd1O/uNO7etz80PgyQFlY27Oa1pQcHSHpK0v6wGJ43iOGDMxgWvC7NG8SwvEEnzJnmcavSzJKBu4GPAvXAHDN7yt0Xd41x969Fjf8KMCHqW+xx9/Hxqk9E5HDMjMLsdAqz0w8ZIOu372HD9lY2Nu9hw/Y9bGhuZeP2PcxctpUtO1vp7HbR7JyMlEhYBOExfHAGQ3IzKMnNoGRwBkNzM8jNSAl9SySeMTYZWO7uKwHM7GHgSmBxD+OvA74Tx3pERHpVdICcURZ7TFtHJ1t27o0Ex/Y9bGxuDZ5HAmX+uu00RZ370T14ztsAAAauSURBVGVQanIQFukMDcJjaFSAlAzOYEhOOqlxPJEwngFRCqyLel0PnBVroJmNAEYCL0U1Z5hZHdAO/MDdn+xh2enAdICKiopeKFtEpPekJidRmheZp+hJa1sHm3e0sqm5lc0797K5uZVNOyKPzc2tzF3TxJYde9nX0XnAcmZQmJXGqKJsfnfzlF6vvb/sCLsWeNTdo3fmjXD39WY2CnjJzBa6+4ruC7r7DGAGRG4Y1Dflioj0nozU5P1nnPfE3WlqaYuESBAem5pb2bKzlXjd9y2eAbEeKI96XRa0xXItcEt0g7uvD76uNLNXiMxPHBQQIiKJwMyCkwjTGDv84MN54yGeV8GaA4w2s5FmlkYkBJ7qPsjMTgHygdlRbflmlh48LwJq6XnuQkRE4iBuWxDu3m5mtwLPETnM9X53X2Rm3wXq3L0rLK4FHvYDb449BrjXzDqJhNgPoo9+EhGR+DOP186rENTU1HhdXV3YZYiInDDMbK6718Tq04XWRUQkJgWEiIjEpIAQEZGYFBAiIhKTAkJERGIaUEcxmVkDsOYYFy8CtvZiOb1N9R0f1Xd8VN/x6c/1jXD34lgdAyogjoeZ1fV0qFd/oPqOj+o7Pqrv+PT3+nqiXUwiIhKTAkJERGJSQHxoRtgFHIbqOz6q7/iovuPT3+uLSXMQIiISk7YgREQkJgWEiIjElHABYWaXmNkHZrbczL4Zoz/dzB4J+t8ys8o+rK3czF42s8VmtsjMbosxZpqZNZvZ/ODx7b6qL3j/1Wa2MHjvgy6daxF3BuvvXTOb2Ie1nRy1Xuab2Q4z+2q3MX26/szsfjPbYmbvRbUVmNkLZrYs+Jrfw7I3BmOWmdmNfVjff5jZ+8HP7wkzy+th2UN+FuJY3z+Z2fqon+FlPSx7yN/1ONb3SFRtq81sfg/Lxn39HTd3T5gHkftSrABGAWnAAmBstzF/BdwTPL8WeKQP6xsGTAye5wBLY9Q3DfhDiOtwNVB0iP7LgGcBA84G3grxZ72JyElAoa0/4DxgIvBeVNu/A98Mnn8T+GGM5QqAlcHX/OB5fh/VdxGQEjz/Yaz6juSzEMf6/gn42yP4+R/ydz1e9XXr/zHw7bDW3/E+Em0LYjKw3N1Xuvs+4GHgym5jrgR+Ezx/FLjQzKwvinP3je4+L3i+E1gClPbFe/eiK4HfesSbQJ6ZDQuhjguBFe5+rGfW9wp3fw1o7NYc/Rn7DfCJGIteDLzg7o3u3gS8AFzSF/W5+/Pu3h68fJPI7YJD0cP6OxJH8rt+3A5VX/D/xjXAQ739vn0l0QKiFFgX9bqeg/8D3j8m+CVpBgr7pLoowa6tCcBbMbqnmNkCM3vWzE7t08LAgefNbK6ZTY/RfyTruC9cS8+/mGGuP4Ch7r4xeL4JGBpjTH9Zj58nskUYy+E+C/F0a7AL7P4edtH1h/V3LrDZ3Zf10B/m+jsiiRYQJwQzywYeA77q7ju6dc8jsttkHHAX8GQfl3eOu08ELgVuMbPz+vj9D8si90D/OPD7GN1hr78DeGRfQ7881tzM/gFoBx7oYUhYn4WfA1XAeGAjkd04/dF1HHrrod//LiVaQKwHyqNelwVtMceYWQowGNjWJ9VF3jOVSDg84O6Pd+939x3uvit4/gyQamZFfVWfu68Pvm4BniCyKR/tSNZxvF0KzHP3zd07wl5/gc1du92Cr1tijAl1PZrZ54DLgeuDEDvIEXwW4sLdN7t7h7t3Ar/o4X3DXn8pwCeBR3oaE9b6OxqJFhBzgNFmNjL4K/Na4KluY54Cuo4YuRp4qadfkN4W7LO8D1ji7nf0MKaka07EzCYT+Rn2SYCZWZaZ5XQ9JzKZ+V63YU8Bnw2OZjobaI7andJXevzLLcz1FyX6M3Yj8L8xxjwHXGRm+cEulIuCtrgzs0uAvwM+7u4tPYw5ks9CvOqLntO6qof3PZLf9Xj6CPC+u9fH6gxz/R2VsGfJ+/pB5CibpUSOcPiHoO27RH4ZADKI7JpYDrwNjOrD2s4hsrvhXWB+8LgMuBm4ORhzK7CIyFEZbwJT+7C+UcH7Lghq6Fp/0fUZcHewfhcCNX38880i8h/+4Ki20NYfkaDaCLQR2Q/+BSJzWi8Cy4A/AQXB2Brgl1HLfj74HC4HburD+pYT2X/f9RnsOqpvOPDMoT4LfVTffwefrXeJ/Kc/rHt9weuDftf7or6g/dddn7mosX2+/o73oUttiIhITIm2i0lERI6QAkJERGJSQIiISEwKCBERiUkBISIiMSkgREQkJgWEiIjE9P8BfLRH1Kr94kkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "F93P9OlMwDbk",
        "outputId": "f5e8444c-c0f9-4c1c-b63c-70d974557239"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(val_accs)\n",
        "plt.ylabel(n_epochs)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8deHLISQBBII+74vLogR1LYWrQu1jjhtfwq1VduOPqy1v9Zu2mUc68z8WjuLU1vHVlt3Kyqtlk5RFLdpVZRgAQnrBQIJWwIhJCFk//z+uCf0GrKS3Nwk9/18PO4j937Pcj/nZHnnfL/nnmPujoiISHv1i3UBIiLSuyg4RESkQxQcIiLSIQoOERHpEAWHiIh0SGKsC+gOQ4cO9QkTJsS6DBGRXmXt2rWH3D27aXtcBMeECRPIzc2NdRkiIr2Kme1url1dVSIi0iFRDQ4zW2hmW80sZGZ3NDN9nJm9bmZ/NbMNZnZ50D7BzI6b2brg8cuIZc42sw+Cdd5nZhbNbRARkQ+LWnCYWQJwP/BJYBawxMxmNZnth8Cz7n4WsBj474hpO9x9TvC4OaL9AeBGYGrwWBitbRARkZNF84hjHhBy953uXgMsBRY1mceBjOD5IGBfays0s5FAhruv9vC1Uh4HruraskVEpDXRDI7RQEHE68KgLdJdwOfNrBBYAXwtYtrEoAvrTTP7WMQ6C9tYJwBmdpOZ5ZpZbnFxcSc2Q0REIsV6cHwJ8Ki7jwEuB54ws37AfmBc0IX1TeC3ZpbRynpO4u4PunuOu+dkZ590NpmIiJyiaJ6OuxcYG/F6TNAW6csEYxTu/o6ZpQBD3b0IqA7a15rZDmBasPyYNtYpIiJRFM0jjjXAVDObaGbJhAe/lzeZZw/wCQAzmwmkAMVmlh0MrmNmkwgPgu909/1AmZmdG5xNdR3whyhug4hITOworuA3f9nF2t1HqG/oWbe/iNoRh7vXmdmtwEogAXjY3fPM7G4g192XA98CHjKz2wgPlN/g7m5mFwB3m1kt0ADc7O4lwapvAR4FBgAvBg8RkT5h/9Hj/GzVdp5bW3giMAYNSOJjU4eyYPowLpg2lGHpKTGt0eLhRk45OTmuT46LSE925FgND7y5g0ffzgeHa88dxxfOHc+m/WW8sbWYN7cVU1xeDcCskRksmJ7Nx6dlM3d8JkkJ0ek8MrO17p5zUruCQ0S62tHjtWwoLCX/cCUJZiQmGEkJRmK/fie+htv6kZQQPD/R9rfpGQOSyEhJisk2uDtVtQ0MSE6I6vscq67jkbd28as3d3Kspo5Pzx3DNy6eypjM1JPq2bS/jDe3FfPG1uITXVjp/RP5yJSh4SCZns3IQQO6rDYFh4JDJCqq6+rZsr+cdQWlrC8oZV1hKTuLj3XJuhP6GQtnj+C688Yzb2IW3XGhiKraev6wbi+Pvb2bTfvLmDtuMJfNHsFls0cwYejALnufmroGlq7Zw32vhjhUUc2ls4bz7cumM214eruWL6uq5e3QoRNBsv9oFQDTh6fz8enZLJiWzdkTMumfeOrBp+BQcIh0mruTf7iSdQVHWF9wlHUFpWzaV0ZNfQMA2en9mTN2MHPGDubMMYOZOjwNd6itb6Cuwamrb6C23qlrCL4G7bX1DdQF7TWN7fXO9qJyns0t5OjxWmaOzOD688azaM7oqBwFFJRU8uTq3TyTW0BpZS0zRqTz8WnZvLXjEBv3lgEwY0T6iRCZOTL9lIKsocFZvn4f//HKVgpKjjN/Yha3f3IGc8dlnnLt7s72ogre2FrEG1uLWZNfQm29k5qcwLKbz2fWqA59muEEBYeCQ6TDDlVUs/7EkcRR1heUcvR4LQCpyQmcPnrQ34Ji7GBGDkrp8qOC4zXhI4BH385ny4FyBg1I4ppzxvKFc8czNiu17RW0wt15e8dhHn07n1c3H8TMuHTWcK4/fwLzI45wCkoqeXnTQVbmHWBNfgnuMC4rlYWnjeCy2cM5a2wm/fq1vt3uzutbi/jpS1vZcqCc2aMy+O7CGVwwdWiX77Nj1XW8s+Mwf95ezPc/NfOUjzoUHAoOkTbtP3qcd3eW8O6uw7y7s4Sdh8JdTv0Mpo/IYM7YQZw5ZjBzxg1mSnYaiVEalG2Ou/PerhIeeyeflXkHaXDnEzOGc/354/nolI798T1WXcfv3y/ksXd2EyqqIGtgMkvmjeXa+eMZNbj1MYJDFdWs2nSQl/IO8FboELX1zrD0/lwyazgLTxvBuZOGnDRYvSa/hJ++tIU1+UeYMCSVb106nU+dPrLNsIk1BYeCQ7rZ0eO1bDtYzpljBpOcGOuLNDRvb+lx3t0ZDonVuw6z+3AlAOkpicyfmMU5E7I4a1wmp43OIDW559y+Z//R4zy1eg9Pv7eHw8dqmJw9kOvPn8Cn544hrX/Lde46dIzH38lnWW4h5dV1nD56ENefP4ErzhhJSlLH/ysvq6rl9S1FrMw7wBtbi6msqScjJZGLZw7n0tkjGDkohfte3c6rW4oYlt6fr188latzxkbtLKiupuBQcEg3Kq2sYfGDq9lyoJz0/ol8fHo2F88czoLp2QxOTY5ZXQUllby7q4TVOw/z7q7DFJQcB8KfE5g3MYv5E7M4d9IQZo7MIKGH/zcM4YHsFR/s57G381lfeJS0/ol89uwxXHfeeCZlpwHhMYU3txXz6Nv5vLmtmKQE4/LTR3L9+RM4a+zgLusmqqqt58/bD7Ey7wCrNh+ktDLcpZeRkshXFkzhhvMnRP0Mra6m4FBwSDcpr6rl8795j837yrj9kzPYfrCcVZuLOFRRTUI/I2d8JpfMGs4nZg5nYheepdOUu1NQcpzVuw6Hg2JnCXtLw0GRmRoOinMnDWH+xCHMGJHe47tN2vLXPUd47O18/vTBfmrrnQumZXPO+Ex+934h+YcrGZben2vnj2fJ/LFR/wBdbX0D7+0qYWdxBVeeOZpBqbE5pbizFBwKDukGx2vquf6R91i7+wi//PzZXDJrOBD+r3fD3qOs2nSQVZsPsuVAOQCTswdy8czhXDxrOHPHZZ7Sf/kNDU7hkeOEissJFVUQKqpge/C1vKoOgCEDk5k/KYv5E4dw7qQhTB2W1uuDoiXF5dU8/d4ennp3NwfLqskZn8l1509g4ewRPbbLsKdScCg4JMqq6+q58fG1/Hl7MT9bfBZXnjmqxXkLSip5bUsRqzYfZPXOw9TWO5mpSVw4YxiXzBzOx6Zln9RXX1PXwO7Dxz4UDKGiCnYeqqCqtuHEfEPT+jN1WBpThqUxfUQ68ydmMWVYWrd8BqInqa1voLi8us3BbmmZgkPBIVFUV9/AV3/7PivzDvLTz5zB1eeMbXuhQHlVLf+77RCrNh/k9a1FlFbWkpzQj/mTspg1MoP8ICx2H66kLuJid6MHD2Dq8DSmZIdDIvw8vdd2i0jP01Jw9JzTJER6qYYG59vPrWdl3kH+6e9mdSg0ANJTkvjUGSP51BkjqatvYO3uI7y6pYhVmw7y9o7DTBiSypRhaSw8bUQ4IIalMyl7YI86y0nii37yRDrB3fnBCxt5Yd0+vnPZdL74kYmdWl9iQj/mTxrC/ElD+P7lM2lo8D47FiG9l0aKRE6Ru/Mvf9rM0+/t4ZYFk/nqhVO6/D0UGtITKThETtG9q7bzm7/s4obzJ/Cdy6bHuhyRbqPgEDkFv3pzB/e9up2rc8Zw5xWz4u6MJYlvCg6RDnrinXx+/OIWrjhjJD/+9BnqTpK4o+AQ6YBlawv5xz/kcfHMYdx7zZxecVkOka4W1eAws4VmttXMQmZ2RzPTx5nZ62b2VzPbYGaXB+2XmNlaM/sg+HpRxDJvBOtcFzyGRXMbRBqt+GA/3122no9MGcIvPje311yoTqSrRe10XDNLAO4HLgEKgTVmttzdN0XM9kPgWXd/wMxmASuACcAh4O/cfZ+ZnQasBEZHLHetu+sTfdJtXt9SxNeX/pW54zJ56LqcU7qSqkhfEc1/meYBIXff6e41wFJgUZN5HGi8NdUgYB+Au//V3fcF7XnAADPrH8VaRVr09o5D3PzkWqaPSOfhL56jD95J3Ivmb8BooCDidSEwv8k8dwEvm9nXgIHAxc2s5zPA++5eHdH2iJnVA78D/sWbuW6Kmd0E3AQwbty4U90G6SXcncqaekqP11JaWcPRylpKj9dSU9dAWv9E0lISSeufSHrwNS0lsV13RVu7+wj/8Fgu47JSefxL88lI0eU8RGL9r9MS4FF3/w8zOw94wsxOc/cGADObDdwDXBqxzLXuvtfM0gkHxxeAx5uu2N0fBB6E8LWqorwd0sXcw1d83X24ktLjNZRW1nI0CIUjlbXB63B76fFajlbWnrjvdXslJ/Q7ESiNoRIZLAOTE3n6vT1kp/fnqX+YT9bA2N1HQ6QniWZw7AUiL9ozJmiL9GVgIYC7v2NmKcBQoMjMxgDPA9e5+47GBdx9b/C13Mx+S7hL7KTgkN6jvsHZWVxB3r4y8vYdZePeMjbtLztxb+tIA5ISyExNYlBqMoMHJDFlWBqDU5MYNCCZwalJ4WnB88GpSSQn9ONYdT3l1bVUVNVRUR1+lFeFHxUR7eVVdew/WhWep6qO8uo6xmQO4PEvzWNYRnTv3yDSm0QzONYAU81sIuHAWAx8rsk8e4BPAI+a2UwgBSg2s8HAn4A73P2txpnNLBEY7O6HzCwJuAJYFcVtkC5WXVfPtgMVbNx3lLx9R8nbV8bm/WUnLguenNiPmSPS+dQZI5k9KoPJ2WlkDQyHRMaApG4flHZ3fbhPpImoBYe715nZrYTPiEoAHnb3PDO7G8h19+XAt4CHzOw2wgPlN7i7B8tNAe40szuDVV4KHANWBqGRQDg0HorWNkjnVNfVs77gbwGxce9RQkUVJy4Nnt4/kVmjMvjcvPHMHpXBaaMHMTl7IIk96DRXhYbIyXQ/Duly7s7/bNjPT17ccuJWpUPTkpk9atCJgJg9KoOxman61LVID6b7cUi3WFdQyj//zybW7j7CjBHpPHDtXOaOz2RYen/99y7SRyg4pEvsKz3OT1/awgvr9jE0rT8/+fTp/J+csbokh0gfpOCQTjlWXcev3tzBg3/eSYPDLQsmc8uFU066X7aI9B367ZZT0tDg/O79Qv5t5VaKyqu54oyR3L5wBmOzUmNdmohEmYJDOuzdnYf55z9tYuPeMs4cO5gHPj+Xs8dnxbosEekmCg5pt92Hj/HjFVt4Ke8AIwel8F/XzOHKM0fpzCiROKPgkDaVVdXyi9dCPPpWPgn9jG9eMo0bPzaJAcm6QqxIPFJwSIvq6ht4ek0B976yjSOVNXxm7hi+c9l0huvyGyJxTcEhzTpYVsVXnlzL+3tKmTcxizuvmMVpowfFuiwR6QEUHHKStbtLuPnJ96moquO/rpnDojmj9OE9ETlBwSEf8tS7u7lreR6jBg/giS/PY8aIjLYXEpG4ouAQIHxBwruW5/H0ewVcMC2bny8+i0GpummRiJxMwSEcLKvi5ifX8tc9pdyyYDLfunS6LhUiIi1ScMS53PwSvvLU+xyrruO/r53L5aePjHVJItLDKTjilLvz1Lt7+NEfw+MZT355PtNHpMe6LBHpBRQccai6rp47X8jjmdwCFkzP5mfXaDxDRNpPwRFnDhwNj2esKyjl1guncNsl0zSeISIdouCII+/tKuGWp97neE0dv/z8XBaepvEMEem4qN7c2cwWmtlWMwuZ2R3NTB9nZq+b2V/NbIOZXR4x7XvBclvN7LL2rlNO5u488U4+n3toNekpibzw1Y8oNETklEXtiMPMEoD7gUuAQmCNmS13900Rs/0QeNbdHzCzWcAKYELwfDEwGxgFrDKzacEyba1TIlTV1vOPL2zkubWFXDRjGPdeM4dBAzSeISKnLppdVfOAkLvvBDCzpcAiIPKPvAONH00eBOwLni8Clrp7NbDLzELB+mjHOiWw/+hxbn5iLesLj/J/L5rCNy6epkugi0inRTM4RgMFEa8LgflN5rkLeNnMvgYMBC6OWHZ1k2VHB8/bWicAZnYTcBPAuHHjOl59L1dVW88XH1lDQUklv/rC2Vw2e0SsSxKRPiKqYxztsAR41N3HAJcDT5hZl9Tk7g+6e46752RnZ3fFKnuVf1+5lS0HyvnF5+YqNESkS0XziGMvMDbi9ZigLdKXgYUA7v6OmaUAQ9tYtq11xr2/bD/Er/+yiy+cO54LZwyLdTki0sdE84hjDTDVzCaaWTLhwe7lTebZA3wCwMxmAilAcTDfYjPrb2YTganAe+1cZ1w7cqyGbz23jsnZA/n+5TNjXY6I9EFRO+Jw9zozuxVYCSQAD7t7npndDeS6+3LgW8BDZnYb4YHyG9zdgTwze5bwoHcd8FV3rwdobp3R2obext35/vMfUHKsht9cf45u7SoiUWHhv9N9W05Ojufm5sa6jKh7LreA7yzbwO0LZ/CVBZNjXY6I9HJmttbdc5q2x3pwXLrI7sPHuGt5HvMnZnHTBZNiXY6I9GEKjj6grr6B255ZR79+xn9eM0fXnhKRqNK1qvqAX7we4v09pdy35CxGDx4Q63JEpI/TEUcv9/6eI/z8tRB/f9ZorjxzVKzLEZE4oODoxSqq67jtmXWMyEjhR4tmx7ocEYkT6qrqxe7+Yx4FJZUsvek8MlJ04UIR6R464uilXtq4n2dzC/nKgsnMm5gV63JEJI4oOHqhg2VV3PH7DzhjzCC+cfG0thcQEelCCo5epqHB+fZz66mubeDea+aQlKBvoYh0L/3V6WUeeTufP28/xA+vmMnk7LRYlyMicUjB0YtsOVDGPS9t4eKZw/ncvPi7x4iI9AwKjl6iqraebyxdR0ZKEvd85nTM9OlwEYkNnY7bS/xbcGOmR754DkPS+se6HBGJYzri6AX+vL2Y3/xlF9edN54Lp+vGTCISWwqOHu7IsRq+/dx6pgxL042ZRKRHUHD0YO7O934fvjHTzxbPISVJN2YSkdhTcPRgy9YW8lLeAb596XRmjxoU63JERIAoB4eZLTSzrWYWMrM7mpl+r5mtCx7bzKw0aL8won2dmVWZ2VXBtEfNbFfEtDnR3IZYOXC0irv/uIn5E7P4h4/pxkwi0nNE7awqM0sA7gcuAQqBNWa23N03Nc7j7rdFzP814Kyg/XVgTtCeBYSAlyNW/x13Xxat2mPN3fnhCx9Q29DAPZ85QzdmEpEeJZpHHPOAkLvvdPcaYCmwqJX5lwBPN9P+WeBFd6+MQo090h837GfV5iK+dcl0JgwdGOtyREQ+JJrBMRooiHhdGLSdxMzGAxOB15qZvJiTA+VfzWxD0NXVpz7UUHKshruW53Hm2MF86aMTY12OiMhJesrg+GJgmbvXRzaa2UjgdGBlRPP3gBnAOUAWcHtzKzSzm8ws18xyi4uLo1N1FPzoj3mUV9XyU3VRiUgPFc3g2AuMjXg9JmhrTnNHFQBXA8+7e21jg7vv97Bq4BHCXWIncfcH3T3H3XOys7NPaQO626ubD/KHdfv46oVTmD4iPdbliIg0K5rBsQaYamYTzSyZcDgsbzqTmc0AMoF3mlnHSeMewVEIFr5Y01XAxi6uOybKqmr5wfMbmT48nVsWTIl1OSIiLYraWVXuXmdmtxLuZkoAHnb3PDO7G8h198YQWQwsdXePXN7MJhA+YnmzyaqfMrNswIB1wM3R2obu9JMXt1BUXsUvv3A2yYk9pQdRRORkUb3IobuvAFY0abuzyeu7Wlg2n2YG0939oq6rsGd4Z8dhfvvuHm782ETmjB0c63JERFqlf21j7HhNPXf8fgPjh6TyzUumx7ocEZE26bLqMXbvqm3sPlzJ0zeey4BkXYtKRHo+HXHE0PqCUn795518bv44zps8JNbliIi0i4IjRmrqGvjusg0MS0/hjk/OiHU5IiLtpq6qGPnvN0JsPVjOwzfkkJGSFOtyRETaTUccMbD1QDn3vx5i0ZxRXDRjeKzLERHpEAVHN6tvcL77uw2kpyRx5xWzYl2OiEiHqauqmz3y1i7WF5Ry35KzGJLWp67PKCJxQkcc3Sj/0DH+/eWtXDxzGH93xshYlyMickoUHN3E3bnj9xtI6tePf7nqdMKX2hIR6X0UHN3k6fcKWL2zhO9/aiYjBqXEuhwRkVOm4OgG+48e58crNnPepCEsPmds2wuIiPRgCo4oc3d++PxGahsa+Mln1EUlIr2fgiPKlq/fx6tbivj2pdMZP0T3DxeR3q/V4DCzQWb2EzPbYmYlZnbYzDYHbbr+dxsOV1Tzoz9uYs7YwXzxI7p/uIj0DW0dcTwLHAEWuHuWuw8BLgzano12cb3dvau2he8f/lndP1xE+o62gmOCu9/j7gcaG9z9gLvfA4yPbmm9X27+ET4yZSjThuv+4SLSd7QVHLvN7LtmduKCSmY23MxuBwqiW1rvVt/g7Dx0jKnD0mJdiohIl2orOK4BhgBvmtkRMysB3gCygKvbWrmZLTSzrWYWMrM7mpl+r5mtCx7bzKw0Ylp9xLTlEe0TzezdYJ3PmFlyO7e1WxUeqaSmroEpCg4R6WNavVaVux8xs0eAV4DV7l7ROM3MFgIvtbSsmSUA9wOXAIXAGjNb7u6bItZ/W8T8XwPOiljFcXef08yq7wHudfelZvZL4MvAA61tRyyEisK7SsEhIn1NW2dV/V/gD8CtwEYzWxQx+f+1se55QMjdd7p7DbAUWNTK/EuAp9uox4CLgGVB02PAVW3UERMngiNb4xsi0re01VV1I3C2u18FLAD+0cy+Hkxr6zSh0Xx4HKQwaDuJmY0HJgKvRTSnmFmuma02s8ZwGAKUuntdO9Z5U7B8bnFxcRuldr1QUQVD0/ozKFU3aRKRvqWty6r3a+yecvd8M1sALAv+0Hfl+aWLgWXuXh/RNt7d95rZJOA1M/sAONreFbr7g8CDADk5Od6FtbZLqLiCKcP0gT8R6XvaOuI4aGYnxhmCELkCGAqc3saye4HICzONCdqas5gm3VTuvjf4upPwgPxZwGFgsJk1Bl5r64wZdydUVKHxDRHpk9oKjuuAA5EN7l7n7tcBF7Sx7BpganAWVDLhcFjedCYzmwFkAu9EtGWaWf/g+VDgI8Amd3fgdeCzwazXEx6D6VGKy6spr6pjSraCQ0T6nlaDw90LIz/812TaW20sW0d4UH0lsBl41t3zzOxuM7syYtbFwNIgFBrNBHLNbD3hoPhJxNlYtwPfNLMQ4TGP37RWRyz87YwqDYyLSN8T1VvHuvsKYEWTtjubvL6rmeXepoWusKDral7XVdn1QsU6FVdE+i5dHTcKQkUVpPVPZHiG7ikuIn2PgiMKdhRXMDl7oO69ISJ9koIjCkJFFUxWN5WI9FEKji5WVlXLwbJqjW+ISJ+l4OhiO05cakTBISJ9k4Kji+nihiLS1yk4uliouILkhH6My0qNdSkiIlGh4OhiO4oqmDA0lcQE7VoR6Zv0162L6RpVItLXKTi6UFVtPXtKKjUwLiJ9moKjC+UfPkaDo89wiEifpuDoQjqjSkTigYKjC4WKKjCDyeqqEpE+TMHRhUJFFYzJHEBKUkKsSxERiRoFRxcKFVVoYFxE+jwFRxepb3B2Hjqm8Q0R6fMUHF2k8EglNXUNCg4R6fMUHF1EZ1SJSLyIanCY2UIz22pmITO7o5np95rZuuCxzcxKg/Y5ZvaOmeWZ2QYzuyZimUfNbFfEcnOiuQ3tdSI4snWfcRHp26J2z3EzSwDuBy4BCoE1Zrbc3Tc1zuPut0XM/zXgrOBlJXCdu283s1HAWjNb6e6lwfTvuPuyaNV+KkJFFQxN68+g1KRYlyIiElXRPOKYB4Tcfae71wBLgUWtzL8EeBrA3be5+/bg+T6gCMiOYq2dFiquYMqwgbEuQ0Qk6qIZHKOBgojXhUHbScxsPDAReK2ZafOAZGBHRPO/Bl1Y95pZ/xbWeZOZ5ZpZbnFx8aluQ7u4uy5uKCJxo6cMji8Glrl7fWSjmY0EngC+6O4NQfP3gBnAOUAWcHtzK3T3B909x91zsrOje7BSXF5NeVWdPsMhInEhmsGxFxgb8XpM0NacxQTdVI3MLAP4E/ADd1/d2O7u+z2sGniEcJdYTIWKwwPjurihiMSDaAbHGmCqmU00s2TC4bC86UxmNgPIBN6JaEsGngcebzoIHhyFYGYGXAVsjNoWtNMOnYorInEkamdVuXudmd0KrAQSgIfdPc/M7gZy3b0xRBYDS93dIxa/GrgAGGJmNwRtN7j7OuApM8sGDFgH3BytbWivUFEFaf0TGZGREutSRESiLmrBAeDuK4AVTdrubPL6rmaWexJ4soV1XtSFJXaJUHEFk7MHEj4IEhHp23rK4HivFiqq0PiGiMQNBUcnlVXVcrCsWuMbIhI3FByddGJgXKfiikicUHB0ki5uKCLxRsHRSaHiCpIT+jEuKzXWpYiIdAsFRyftKKpgwtBUEhO0K0UkPuivXSfpGlUiEm8UHJ1QVVvPnpJKDYyLSFxRcHRC/uFjNLiuUSUi8UXB0Qk6o0pE4pGCoxNCRRWYwWR1VYlIHFFwdEKoqIIxmQNISUqIdSkiIt1GwdEJoaIKDYyLSNxRcJyi+gZn56FjGt8Qkbij4DhFhUcqqalrUHCISNxRcJwinVElIvFKwXGKTgRHdnqMKxER6V4KjlO0o7iCoWn9GZSaFOtSRES6VVSDw8wWmtlWMwuZ2R3NTL/XzNYFj21mVhox7Xoz2x48ro9oP9vMPgjWeZ/F6H6t4WtUDYzFW4uIxFTUgsPMEoD7gU8Cs4AlZjYrch53v83d57j7HODnwO+DZbOAfwLmA/OAfzKzzGCxB4AbganBY2G0tqEl7q6LG4pI3IrmEcc8IOTuO929BlgKLGpl/iXA08Hzy4BX3L3E3Y8ArwALzWwkkOHuq93dgceBq6K3Cc0rrqimrKpOnxgXkbgUzeAYDRREvC4M2k5iZuOBicBrbSw7OnjennXeZGa5ZpZbXFx8ShvQEp1RJSLxrKcMji8Glrl7fVet0N0fdPccd8/Jzs7uqtUCEfcZV3CISByKZnDsBcZGvB4TtDVnMX/rpmpt2b3B8/asM2pCRRWk9U9kREZKd7+1iEjMRTM41gBTzWyimSUTDoYrdGEAAAmQSURBVIflTWcysxlAJvBORPNK4FIzywwGxS8FVrr7fqDMzM4Nzqa6DvhDFLehWaHiCiZnDyRGJ3SJiMRU1ILD3euAWwmHwGbgWXfPM7O7zezKiFkXA0uDwe7GZUuAfyYcPmuAu4M2gFuAXwMhYAfwYrS2oSWhogrdvElE4lZiNFfu7iuAFU3a7mzy+q4Wln0YeLiZ9lzgtK6rsmPKqmo5WFat8Q0RiVs9ZXC81zgxMK5TcUUkTik4Okin4opIvFNwdFCouILkhH6My0qNdSkiIjGh4OigHUUVTBiaSmKCdp2IxCf99esgXaNKROKdgqMDqmrr2VNSqYFxEYlrCo4OyD98jAZHn+EQkbim4OgAnVElIqLg6JBQUQVm6HLqIhLXFBwdECqqYEzmAFKSEmJdiohIzCg4OiBUVKGBcRGJewqOdqpvcHYdOqbxDRGJewqOdtp75DjVdQ0KDhGJewqOdgoVlwM6o0pERMHRTidOxc1Oj3ElIiKxpeBop1BRBUPT+jMoNSnWpYiIxJSCo51CReHbxYqIxLuoBoeZLTSzrWYWMrM7WpjnajPbZGZ5ZvbboO1CM1sX8agys6uCaY+a2a6IaXOiuQ0A7q6LG4qIBKJ261gzSwDuBy4BCoE1Zrbc3TdFzDMV+B7wEXc/YmbDANz9dWBOME8W4fuLvxyx+u+4+7Jo1d5UcUU1ZVV1Cg4REaJ7xDEPCLn7TnevAZYCi5rMcyNwv7sfAXD3ombW81ngRXevjGKtrdI1qkRE/iaawTEaKIh4XRi0RZoGTDOzt8xstZktbGY9i4Gnm7T9q5ltMLN7zax/15XcvB0KDhGRE2I9OJ4ITAUWAEuAh8xscONEMxsJnA6sjFjme8AM4BwgC7i9uRWb2U1mlmtmucXFxZ0qMlRUQVr/REZkpHRqPSIifUE0g2MvMDbi9ZigLVIhsNzda919F7CNcJA0uhp43t1rGxvcfb+HVQOPEO4SO4m7P+juOe6ek52d3akNCRWHz6gys06tR0SkL4hmcKwBpprZRDNLJtzltLzJPC8QPtrAzIYS7rraGTF9CU26qYKjECz8V/wqYGM0io8UKqrQzZtERAJRO6vK3evM7FbC3UwJwMPunmdmdwO57r48mHapmW0C6gmfLXUYwMwmED5iebPJqp8ys2zAgHXAzdHaBoCyqloOllVrfENEJBC14ABw9xXAiiZtd0Y8d+CbwaPpsvmcPJiOu1/U5YW24sTAuC6nLiICxH5wvMfTqbgiIh+m4GhDqLiC5IR+jMtKjXUpIiI9goKjDTuKKpgwNJXEBO0qERFQcLRJ16gSEfkwBUcrqmrr2VNSqYFxEZEICo5W5B8+RoOjz3CIiERQcLRCZ1SJiJxMwdGKHUXHMIPJ6qoSETlBwdGKUHEFYzIHkJKUEOtSRER6jKh+cry3mzEinTGZA2JdhohIj6LgaMVXL5wS6xJERHocdVWJiEiHKDhERKRDFBwiItIhCg4REekQBYeIiHSIgkNERDpEwSEiIh2i4BARkQ6x8G2/+zYzKwZ2n+LiQ4FDXVhOV1N9naP6Okf1dU5Pr2+8u2c3bYyL4OgMM8t195xY19ES1dc5qq9zVF/n9PT6WqKuKhER6RAFh4iIdIiCo20PxrqANqi+zlF9naP6Oqen19csjXGIiEiH6IhDREQ6RMEhIiIdouAImNlCM9tqZiEzu6OZ6f3N7Jlg+rtmNqEbaxtrZq+b2SYzyzOzrzczzwIzO2pm64LHnd1VX/D++Wb2QfDeuc1MNzO7L9h/G8xsbjfWNj1iv6wzszIz+0aTebp1/5nZw2ZWZGYbI9qyzOwVM9sefM1sYdnrg3m2m9n13Vjfv5nZluD797yZDW5h2VZ/FqJY311mtjfie3h5C8u2+rsexfqeiagt38zWtbBs1Pdfp7l73D+ABGAHMAlIBtYDs5rMcwvwy+D5YuCZbqxvJDA3eJ4ObGumvgXA/8RwH+YDQ1uZfjnwImDAucC7MfxeHyD8waaY7T/gAmAusDGi7afAHcHzO4B7mlkuC9gZfM0Mnmd2U32XAonB83uaq689PwtRrO8u4Nvt+P63+rserfqaTP8P4M5Y7b/OPnTEETYPCLn7TnevAZYCi5rMswh4LHi+DPiEmVl3FOfu+939/eB5ObAZGN0d792FFgGPe9hqYLCZjYxBHZ8Adrj7qV5JoEu4+/8CJU2aI3/GHgOuambRy4BX3L3E3Y8ArwALu6M+d3/Z3euCl6uBMV39vu3Vwv5rj/b8rndaa/UFfzeuBp7u6vftLgqOsNFAQcTrQk7+w3xinuCX5ygwpFuqixB0kZ0FvNvM5PPMbL2ZvWhms7u1MHDgZTNba2Y3NTO9Pfu4Oyym5V/YWO4/gOHuvj94fgAY3sw8PWU/fonwEWRz2vpZiKZbg660h1vo6usJ++9jwEF3397C9Fjuv3ZRcPQiZpYG/A74hruXNZn8PuHulzOBnwMvdHN5H3X3ucAnga+a2QXd/P5tMrNk4ErguWYmx3r/fYiH+yx65LnyZvYDoA54qoVZYvWz8AAwGZgD7CfcHdQTLaH1o40e/7uk4AjbC4yNeD0maGt2HjNLBAYBh7uluvB7JhEOjafc/fdNp7t7mbtXBM9XAElmNrS76nP3vcHXIuB5wl0Ckdqzj6Ptk8D77n6w6YRY77/Awcbuu+BrUTPzxHQ/mtkNwBXAtUG4naQdPwtR4e4H3b3e3RuAh1p431jvv0Tg08AzLc0Tq/3XEQqOsDXAVDObGPxXuhhY3mSe5UDjGSyfBV5r6RenqwV9or8BNrv7f7Ywz4jGMRczm0f4e9stwWZmA80svfE54UHUjU1mWw5cF5xddS5wNKJbpru0+J9eLPdfhMifseuBPzQzz0rgUjPLDLpiLg3aos7MFgLfBa5098oW5mnPz0K06oscM/v7Ft63Pb/r0XQxsMXdC5ubGMv91yGxHp3vKQ/CZ/1sI3zGxQ+CtrsJ/5IApBDu4ggB7wGTurG2jxLuttgArAselwM3AzcH89wK5BE+S2Q1cH431jcpeN/1QQ2N+y+yPgPuD/bvB0BON39/BxIOgkERbTHbf4QDbD9QS7if/cuEx8xeBbYDq4CsYN4c4NcRy34p+DkMAV/sxvpChMcHGn8GG88yHAWsaO1noZvqeyL42dpAOAxGNq0veH3S73p31Be0P9r4Mxcxb7fvv84+dMkRERHpEHVViYhIhyg4RESkQxQcIiLSIQoOERHpEAWHiIh0iIJDREQ6RMEhIiId8v8B7BaMJ8Yrj1cAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jHt6rTougG0K"
      },
      "source": [
        "### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYvpk9VHb4CD"
      },
      "source": [
        "test_data = wsd_data['test']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XjSdDIGTf3a"
      },
      "source": [
        "**Normal version of classifier**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHYxcJBNexzB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f88194d0-d169-4a75-ac61-b70f6246aafd"
      },
      "source": [
        "# TODO HERE : run on dev and evaluate\n",
        "pred_labels, val_losses, dev_acc, b_labels, _ = classifier.run_on_dataset(dev_data, optimizer, batch_size=32, validation_use=False)\n",
        "pred_labels, val_losses, test_acc, b_labels, _ = classifier.run_on_dataset(test_data, optimizer, batch_size=32, validation_use=False)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:68: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2VHBqVMXT0WW",
        "outputId": "1e0c7e43-1c79-4ebb-fbe4-501abb5dee69"
      },
      "source": [
        "print('dev acc: ', dev_acc, 'test acc:', test_acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dev acc:  0.8452380952380952 test acc: 0.8507070249597424\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZH92iHUaTjGA"
      },
      "source": [
        "**Classifier with mlp, lemmas and weights**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unUsvJIJb_M5"
      },
      "source": [
        "pred_labels, val_losses, dev_acc1, b_labels, _ = classifier_mlp_weights_lemmas.run_on_dataset(dev_data, optimizer, batch_size=32, validation_use=False)\n",
        "pred_labels, val_losses, test_acc1, b_labels, _ = classifier_mlp_weights_lemmas.run_on_dataset(test_data, optimizer, batch_size=32, validation_use=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y6RvXGnuT9qE",
        "outputId": "1651c3d4-2b10-47b8-b221-0fcceac76a00"
      },
      "source": [
        "print('dev acc: ', dev_acc1, 'test acc:', test_acc1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dev acc:  0.8571428571428571 test acc: 0.8703829508856683\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "58FEY1qRTnlh"
      },
      "source": [
        "**Classifier with weights**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ex411WDicACm",
        "outputId": "e6e32df1-9148-4628-ca28-9b7c02436623"
      },
      "source": [
        "pred_labels, val_losses, dev_acc2, b_labels, _ = classifier_weights.run_on_dataset(dev_data, optimizer, batch_size=32, validation_use=False)\n",
        "pred_labels, val_losses, test_acc2, b_labels, _ = classifier_weights.run_on_dataset(test_data, optimizer, batch_size=32, validation_use=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:68: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fGjo_stpUAT7",
        "outputId": "135a38a9-5ca7-4ec9-a8b9-502f739b1e7a"
      },
      "source": [
        "print('dev acc: ', dev_acc2, 'test acc:' , test_acc2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dev acc:  0.8519345238095238 test acc: 0.8606078904991948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5KzKpherTqsj"
      },
      "source": [
        "**Classifier mlp and weights**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I7tpF6H5cArk",
        "outputId": "ae365b05-0e8f-420e-90ff-bb064f79109a"
      },
      "source": [
        "pred_labels, val_losses, dev_acc3, b_labels, _ = classifier_mlp_weights.run_on_dataset(dev_data, optimizer, batch_size=32, validation_use=False)\n",
        "pred_labels, val_losses, test_acc3, b_labels, _ = classifier_mlp_weights.run_on_dataset(test_data, optimizer, batch_size=32, validation_use=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:68: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zwrX4nJoUCKS",
        "outputId": "2b07eccf-5fc6-4bdf-bf53-9d253a48c823"
      },
      "source": [
        "print('dev acc: ', dev_acc3, 'test acc:', test_acc3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dev acc:  0.8586309523809523 test acc: 0.862746578099839\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--HB8b_iWmU_"
      },
      "source": [
        "**I solved the problem of UserWarning, it's still present in the output cells because I didn't want to retrain everything**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BmIAkHHqSbXZ"
      },
      "source": [
        "# BONUS : generalization analysis\n",
        "#   Do you think it would be better to predict seen-in-train lemma/frame associations only ?\n",
        "#   (implement analysis of the predictions to answer that question)\n",
        "\n",
        "# VARIOUS OTHER POSSIBLE BONUSES: does it help to:\n",
        "# - fine-tune with a MLP instead of single layer ? + ADDED\n",
        "# - balance classes (\"Other_sense\" is over represented in dataset) ?+ ADDED\n",
        "#    Not sure, because natural distribution of data is a precious clue (cf. MFS)\n",
        "# - add a lemma embedding of the target ? - nn_embeddings ? + ADDED\n",
        "# - use the average of the target subword tokens instead of the first one only ?\n",
        "# ... other ideas are welcome ...\n",
        "#ADDED also early stopping \n",
        "\n",
        "\n",
        "#@@ Très bien pour les bonus"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}