Translation attempt.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPbEwT3QBxAuR/ebs56xONQ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "082da5ca27a34904b335835c0ee83107": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6f256e96c0244520a517d8a687d201ac",
              "IPY_MODEL_68b835d96502425ebb4bb4f69b15e2f2",
              "IPY_MODEL_ba43dacc566d4ed49f000507e0053053"
            ],
            "layout": "IPY_MODEL_e9e36355aa134025b31baa241a3ccd5b"
          }
        },
        "6f256e96c0244520a517d8a687d201ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_02b63f5192d741cdbc1e71f1d230e16a",
            "placeholder": "​",
            "style": "IPY_MODEL_b1009fd0a79d4fe19ab6c305fd15cc1d",
            "value": "config.json: 100%"
          }
        },
        "68b835d96502425ebb4bb4f69b15e2f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_43e72968d9b64c8f99d41945ed70cb39",
            "max": 1417,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d0291c48231b4b249be66758c033ff7a",
            "value": 1417
          }
        },
        "ba43dacc566d4ed49f000507e0053053": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df757985d1154bd593f43b2c56a5d839",
            "placeholder": "​",
            "style": "IPY_MODEL_903200fa21214ecd9a9082921cff03d0",
            "value": " 1.42k/1.42k [00:00&lt;00:00, 56.0kB/s]"
          }
        },
        "e9e36355aa134025b31baa241a3ccd5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02b63f5192d741cdbc1e71f1d230e16a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1009fd0a79d4fe19ab6c305fd15cc1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "43e72968d9b64c8f99d41945ed70cb39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d0291c48231b4b249be66758c033ff7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "df757985d1154bd593f43b2c56a5d839": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "903200fa21214ecd9a9082921cff03d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2e731ad3d53b4020a049a2074c37646e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5a5e0c7922f5480791676ef5d5dcd396",
              "IPY_MODEL_3ce41dd0ec78401d9ece4e454a078c94",
              "IPY_MODEL_4427a27efcf34633bc711117c504df5c"
            ],
            "layout": "IPY_MODEL_9015c45dfaaf42e48665d7472009fafd"
          }
        },
        "5a5e0c7922f5480791676ef5d5dcd396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3054f715cb87412cbb9c89f2a5826b6c",
            "placeholder": "​",
            "style": "IPY_MODEL_126a1d6af7704a05ae6234fc1449124e",
            "value": "model.safetensors: 100%"
          }
        },
        "3ce41dd0ec78401d9ece4e454a078c94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9eff74c8b0fb4ccba4c7f07b527daaaf",
            "max": 1112264584,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f1222f7af43642cbb372b1329d541b86",
            "value": 1112264584
          }
        },
        "4427a27efcf34633bc711117c504df5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c8f16b4a75646c08d692161fc060721",
            "placeholder": "​",
            "style": "IPY_MODEL_6cc3655bfd3d41ecb203992ef6cda6e1",
            "value": " 1.11G/1.11G [00:06&lt;00:00, 59.1MB/s]"
          }
        },
        "9015c45dfaaf42e48665d7472009fafd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3054f715cb87412cbb9c89f2a5826b6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "126a1d6af7704a05ae6234fc1449124e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9eff74c8b0fb4ccba4c7f07b527daaaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1222f7af43642cbb372b1329d541b86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2c8f16b4a75646c08d692161fc060721": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cc3655bfd3d41ecb203992ef6cda6e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1cd16c55340d42e2984c85b8203dfdca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0bcef8c3d2ab4eaea5e6440c8128159d",
              "IPY_MODEL_c435fdee5aff4bb9a94d4e85db0e83ce",
              "IPY_MODEL_ac88344ca11440ca882e262c8a11de19"
            ],
            "layout": "IPY_MODEL_a7d74b926a2f4e8fa59ffe106feacc3d"
          }
        },
        "0bcef8c3d2ab4eaea5e6440c8128159d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac7f8a9e113e450ca02558e569d6536d",
            "placeholder": "​",
            "style": "IPY_MODEL_3eba35d753e04a2db16b9a5d9332fe2d",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "c435fdee5aff4bb9a94d4e85db0e83ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fdeac44ad3f74e078746efa51e0aacc7",
            "max": 502,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7703afeadd4b421499ff1349fb5e0015",
            "value": 502
          }
        },
        "ac88344ca11440ca882e262c8a11de19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_690b1692fb0749aca0f1fbf674c72c86",
            "placeholder": "​",
            "style": "IPY_MODEL_14ab695850be402092dd66aa07b3f570",
            "value": " 502/502 [00:00&lt;00:00, 29.4kB/s]"
          }
        },
        "a7d74b926a2f4e8fa59ffe106feacc3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac7f8a9e113e450ca02558e569d6536d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3eba35d753e04a2db16b9a5d9332fe2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fdeac44ad3f74e078746efa51e0aacc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7703afeadd4b421499ff1349fb5e0015": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "690b1692fb0749aca0f1fbf674c72c86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14ab695850be402092dd66aa07b3f570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a9b73a18e3249f0ad8ed6d459de7ebe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f4d2791017ad42859429900e893740de",
              "IPY_MODEL_d69c2b1b1bd64a109a5c29532ed4319a",
              "IPY_MODEL_0cb384afc19643f7a0ef98cdaee7dd06"
            ],
            "layout": "IPY_MODEL_3c3f5b36d33543e4bf8412f0da71b176"
          }
        },
        "f4d2791017ad42859429900e893740de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a38261c3aae14b7e8570bf46d30026dc",
            "placeholder": "​",
            "style": "IPY_MODEL_8dc27996ab904320abfc0c1563554a6d",
            "value": "sentencepiece.bpe.model: 100%"
          }
        },
        "d69c2b1b1bd64a109a5c29532ed4319a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_501d891f161e4d7198c683b01b3902a6",
            "max": 5069051,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6a3d3d82ab984041884d4cf5b7f76a6e",
            "value": 5069051
          }
        },
        "0cb384afc19643f7a0ef98cdaee7dd06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44e38e1e57e045a096d75a2136c5b4c4",
            "placeholder": "​",
            "style": "IPY_MODEL_f0988aaab790490c80da41aa9e92fc16",
            "value": " 5.07M/5.07M [00:00&lt;00:00, 46.6MB/s]"
          }
        },
        "3c3f5b36d33543e4bf8412f0da71b176": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a38261c3aae14b7e8570bf46d30026dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dc27996ab904320abfc0c1563554a6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "501d891f161e4d7198c683b01b3902a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a3d3d82ab984041884d4cf5b7f76a6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "44e38e1e57e045a096d75a2136c5b4c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0988aaab790490c80da41aa9e92fc16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e83d4ca97cbd47709da86f96a414e4fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0d7ae2ec180c43c88ae4045b48d23406",
              "IPY_MODEL_5249ff5562db47f78086fc14b53887f2",
              "IPY_MODEL_811d26a54646479c91ba6a1368feb6db"
            ],
            "layout": "IPY_MODEL_df9dced3d92d4c3c9ed7af4e3c19b080"
          }
        },
        "0d7ae2ec180c43c88ae4045b48d23406": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b663c520c634ea9ac7dcc7ecfa41e5e",
            "placeholder": "​",
            "style": "IPY_MODEL_8d0c757c419047f298578c08ad36606d",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "5249ff5562db47f78086fc14b53887f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0bb075a2e1d495e81d32d6df4798f49",
            "max": 239,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_365098cb0e6a49648ffa13e51757e441",
            "value": 239
          }
        },
        "811d26a54646479c91ba6a1368feb6db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd90f83f47a8480991ad482609f32518",
            "placeholder": "​",
            "style": "IPY_MODEL_4c06e1930705449bac6e2dda35e73279",
            "value": " 239/239 [00:00&lt;00:00, 16.8kB/s]"
          }
        },
        "df9dced3d92d4c3c9ed7af4e3c19b080": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b663c520c634ea9ac7dcc7ecfa41e5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d0c757c419047f298578c08ad36606d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f0bb075a2e1d495e81d32d6df4798f49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "365098cb0e6a49648ffa13e51757e441": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd90f83f47a8480991ad482609f32518": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c06e1930705449bac6e2dda35e73279": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f620745f78248659008637587c9665f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1547885731d74c1ba4238dfca5196031",
              "IPY_MODEL_d6e8221c01ce4157af526957d103caed",
              "IPY_MODEL_c363ec2a0819464fb34780f45041baa4"
            ],
            "layout": "IPY_MODEL_c36926fb82aa43238338a9959d87dec3"
          }
        },
        "1547885731d74c1ba4238dfca5196031": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36e2a0bd20564542b17f0bb7c56f9f6c",
            "placeholder": "​",
            "style": "IPY_MODEL_9c2428f0e7b2404395d2a6291903ba47",
            "value": "tokenizer.json: 100%"
          }
        },
        "d6e8221c01ce4157af526957d103caed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_627f34231b4942c2af26e8b258341e3f",
            "max": 9081351,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d23cc934de664c43b33de454c5dd62f9",
            "value": 9081351
          }
        },
        "c363ec2a0819464fb34780f45041baa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae0c1dce60dd4ff7b60095afd79ee569",
            "placeholder": "​",
            "style": "IPY_MODEL_446b0a5f0c9d4a2ab0d05cfe19a3235d",
            "value": " 9.08M/9.08M [00:00&lt;00:00, 62.1MB/s]"
          }
        },
        "c36926fb82aa43238338a9959d87dec3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36e2a0bd20564542b17f0bb7c56f9f6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c2428f0e7b2404395d2a6291903ba47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "627f34231b4942c2af26e8b258341e3f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d23cc934de664c43b33de454c5dd62f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ae0c1dce60dd4ff7b60095afd79ee569": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "446b0a5f0c9d4a2ab0d05cfe19a3235d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ontenkutsenko/NLP_Airbnb_Delisting_Prediction/blob/main/Translation%20attempt.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "Bwd4VBQfuW4g"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "-4B10QSUsy31"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!gdown 1Y63uPJCSWvlefU-HYpuVumaiqgjHjpb8\n",
        "!unzip '/content/Project Corpora.zip'\n",
        "!rm '/content/Project Corpora.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zYvLw_OHt_Dg",
        "outputId": "379de140-9b70-4ba6-ec43-524bee1bc9e8"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1Y63uPJCSWvlefU-HYpuVumaiqgjHjpb8\n",
            "From (redirected): https://drive.google.com/uc?id=1Y63uPJCSWvlefU-HYpuVumaiqgjHjpb8&confirm=t&uuid=2cb1302a-2134-4bad-9993-3fd55afb372c\n",
            "To: /content/Project Corpora.zip\n",
            "100% 55.4M/55.4M [00:00<00:00, 73.0MB/s]\n",
            "Archive:  /content/Project Corpora.zip\n",
            "  inflating: Project Corpora/test.xlsx  \n",
            "  inflating: Project Corpora/test_reviews.xlsx  \n",
            "  inflating: Project Corpora/train.xlsx  \n",
            "  inflating: Project Corpora/train_reviews.xlsx  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_train = pd.read_excel('/content/Project Corpora/train.xlsx')\n",
        "df_train_reviews = pd.read_excel('/content/Project Corpora/train_reviews.xlsx')\n",
        "df_test = pd.read_excel('/content/Project Corpora/test.xlsx')\n",
        "df_test_reviews = pd.read_excel('/content/Project Corpora/test_reviews.xlsx')"
      ],
      "metadata": {
        "id": "0d4roZbou3OF"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, XLMRobertaForSequenceClassification, XLMRobertaTokenizer\n",
        "\n",
        "# Load the pre-trained model checkpoint and tokenizer\n",
        "model = XLMRobertaForSequenceClassification.from_pretrained(\"papluca/xlm-roberta-base-language-detection\")\n",
        "tokenizer = XLMRobertaTokenizer.from_pretrained(\"papluca/xlm-roberta-base-language-detection\")\n",
        "\n",
        "\n",
        "pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, device=0)  # Specify GPU device if available\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333,
          "referenced_widgets": [
            "082da5ca27a34904b335835c0ee83107",
            "6f256e96c0244520a517d8a687d201ac",
            "68b835d96502425ebb4bb4f69b15e2f2",
            "ba43dacc566d4ed49f000507e0053053",
            "e9e36355aa134025b31baa241a3ccd5b",
            "02b63f5192d741cdbc1e71f1d230e16a",
            "b1009fd0a79d4fe19ab6c305fd15cc1d",
            "43e72968d9b64c8f99d41945ed70cb39",
            "d0291c48231b4b249be66758c033ff7a",
            "df757985d1154bd593f43b2c56a5d839",
            "903200fa21214ecd9a9082921cff03d0",
            "2e731ad3d53b4020a049a2074c37646e",
            "5a5e0c7922f5480791676ef5d5dcd396",
            "3ce41dd0ec78401d9ece4e454a078c94",
            "4427a27efcf34633bc711117c504df5c",
            "9015c45dfaaf42e48665d7472009fafd",
            "3054f715cb87412cbb9c89f2a5826b6c",
            "126a1d6af7704a05ae6234fc1449124e",
            "9eff74c8b0fb4ccba4c7f07b527daaaf",
            "f1222f7af43642cbb372b1329d541b86",
            "2c8f16b4a75646c08d692161fc060721",
            "6cc3655bfd3d41ecb203992ef6cda6e1",
            "1cd16c55340d42e2984c85b8203dfdca",
            "0bcef8c3d2ab4eaea5e6440c8128159d",
            "c435fdee5aff4bb9a94d4e85db0e83ce",
            "ac88344ca11440ca882e262c8a11de19",
            "a7d74b926a2f4e8fa59ffe106feacc3d",
            "ac7f8a9e113e450ca02558e569d6536d",
            "3eba35d753e04a2db16b9a5d9332fe2d",
            "fdeac44ad3f74e078746efa51e0aacc7",
            "7703afeadd4b421499ff1349fb5e0015",
            "690b1692fb0749aca0f1fbf674c72c86",
            "14ab695850be402092dd66aa07b3f570",
            "3a9b73a18e3249f0ad8ed6d459de7ebe",
            "f4d2791017ad42859429900e893740de",
            "d69c2b1b1bd64a109a5c29532ed4319a",
            "0cb384afc19643f7a0ef98cdaee7dd06",
            "3c3f5b36d33543e4bf8412f0da71b176",
            "a38261c3aae14b7e8570bf46d30026dc",
            "8dc27996ab904320abfc0c1563554a6d",
            "501d891f161e4d7198c683b01b3902a6",
            "6a3d3d82ab984041884d4cf5b7f76a6e",
            "44e38e1e57e045a096d75a2136c5b4c4",
            "f0988aaab790490c80da41aa9e92fc16",
            "e83d4ca97cbd47709da86f96a414e4fe",
            "0d7ae2ec180c43c88ae4045b48d23406",
            "5249ff5562db47f78086fc14b53887f2",
            "811d26a54646479c91ba6a1368feb6db",
            "df9dced3d92d4c3c9ed7af4e3c19b080",
            "4b663c520c634ea9ac7dcc7ecfa41e5e",
            "8d0c757c419047f298578c08ad36606d",
            "f0bb075a2e1d495e81d32d6df4798f49",
            "365098cb0e6a49648ffa13e51757e441",
            "cd90f83f47a8480991ad482609f32518",
            "4c06e1930705449bac6e2dda35e73279",
            "0f620745f78248659008637587c9665f",
            "1547885731d74c1ba4238dfca5196031",
            "d6e8221c01ce4157af526957d103caed",
            "c363ec2a0819464fb34780f45041baa4",
            "c36926fb82aa43238338a9959d87dec3",
            "36e2a0bd20564542b17f0bb7c56f9f6c",
            "9c2428f0e7b2404395d2a6291903ba47",
            "627f34231b4942c2af26e8b258341e3f",
            "d23cc934de664c43b33de454c5dd62f9",
            "ae0c1dce60dd4ff7b60095afd79ee569",
            "446b0a5f0c9d4a2ab0d05cfe19a3235d"
          ]
        },
        "id": "HN1xWU-dG83-",
        "outputId": "772945b2-2ba5-4f96-ebe7-ce976786003a"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.42k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "082da5ca27a34904b335835c0ee83107"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2e731ad3d53b4020a049a2074c37646e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/502 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1cd16c55340d42e2984c85b8203dfdca"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3a9b73a18e3249f0ad8ed6d459de7ebe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/239 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e83d4ca97cbd47709da86f96a414e4fe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/9.08M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f620745f78248659008637587c9665f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = df_train_reviews.comments.to_list()[:10000]\n",
        "labels = []\n",
        "\n",
        "for i in tqdm(text):\n",
        "  labels.append(pipe(i, top_k=1, truncation=True)[0]['label'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Od65MzPEG-Jc",
        "outputId": "662b8279-05ce-4770-8c0c-98c9e3887fa1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 10/10000 [00:02<22:06,  7.53it/s] You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "100%|██████████| 10000/10000 [02:00<00:00, 83.20it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment = df_train_reviews[:10000]"
      ],
      "metadata": {
        "id": "bRL2WgxtMrtH"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment['language'] = labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EIgu8ELwMbcI",
        "outputId": "89b9428a-ba24-47dd-8598-b92a9bf88d83"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-8-5f11783fc29e>:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_experiment['language'] = labels\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment = df_experiment.sort_values(by = 'language')"
      ],
      "metadata": {
        "id": "9NBhQLRZM6RS"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment.language.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "88XT20bKQjaX",
        "outputId": "089e25f1-a8bc-477e-b24c-c2666068d925"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "language\n",
              "en    6353\n",
              "fr    1482\n",
              "pt     728\n",
              "es     596\n",
              "de     380\n",
              "it     160\n",
              "nl     101\n",
              "ru      64\n",
              "ja      42\n",
              "zh      30\n",
              "ur      22\n",
              "pl      14\n",
              "hi      12\n",
              "tr       9\n",
              "sw       4\n",
              "bg       1\n",
              "el       1\n",
              "ar       1\n",
              "Name: count, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_ex = df_experiment.copy()"
      ],
      "metadata": {
        "id": "qIe8SxUkTXys"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment = df_experiment[df_experiment['language']=='de']"
      ],
      "metadata": {
        "id": "2A6QA-LRTbev"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "tjuNNo9ETezr",
        "outputId": "baccf055-eea8-4df4-c07e-9f412b4703a0"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      index                                           comments language\n",
              "8132    171                         Der ideale Ort zum Rekaxen       de\n",
              "9603    204  Wir sind zum ersten Mal as Familie mit zwei gr...       de\n",
              "9845    207  Die Wohung von Clement liegt perfekt, nur weni...       de\n",
              "1569     30  Der Garten und der Pool sind ein Traum. Den Po...       de\n",
              "5711    119  Gerne wieder beim nächsten Aufenthalt in Lissabon       de\n",
              "...     ...                                                ...      ...\n",
              "9071    194  Tolle Lage und schöne Wohnung. Viel schöne Res...       de\n",
              "7580    161  Wir haben uns im Castro Beach House wirklich s...       de\n",
              "6883    146  Die Kommunikation war super alles hat schnell ...       de\n",
              "5140    100  angenehm und unkompliziert. die wohnung ist ei...       de\n",
              "1725     35  Alles war perfekt: die Gastgeber, das Haus, di...       de\n",
              "\n",
              "[380 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a7c3dffd-30f6-4688-9280-2fe452e425d4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>comments</th>\n",
              "      <th>language</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8132</th>\n",
              "      <td>171</td>\n",
              "      <td>Der ideale Ort zum Rekaxen</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9603</th>\n",
              "      <td>204</td>\n",
              "      <td>Wir sind zum ersten Mal as Familie mit zwei gr...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9845</th>\n",
              "      <td>207</td>\n",
              "      <td>Die Wohung von Clement liegt perfekt, nur weni...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1569</th>\n",
              "      <td>30</td>\n",
              "      <td>Der Garten und der Pool sind ein Traum. Den Po...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5711</th>\n",
              "      <td>119</td>\n",
              "      <td>Gerne wieder beim nächsten Aufenthalt in Lissabon</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9071</th>\n",
              "      <td>194</td>\n",
              "      <td>Tolle Lage und schöne Wohnung. Viel schöne Res...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7580</th>\n",
              "      <td>161</td>\n",
              "      <td>Wir haben uns im Castro Beach House wirklich s...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6883</th>\n",
              "      <td>146</td>\n",
              "      <td>Die Kommunikation war super alles hat schnell ...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5140</th>\n",
              "      <td>100</td>\n",
              "      <td>angenehm und unkompliziert. die wohnung ist ei...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1725</th>\n",
              "      <td>35</td>\n",
              "      <td>Alles war perfekt: die Gastgeber, das Haus, di...</td>\n",
              "      <td>de</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>380 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a7c3dffd-30f6-4688-9280-2fe452e425d4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a7c3dffd-30f6-4688-9280-2fe452e425d4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a7c3dffd-30f6-4688-9280-2fe452e425d4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c770ef3c-68d9-44a2-9b13-ca58aecb32fa\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c770ef3c-68d9-44a2-9b13-ca58aecb32fa')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c770ef3c-68d9-44a2-9b13-ca58aecb32fa button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_experiment",
              "summary": "{\n  \"name\": \"df_experiment\",\n  \"rows\": 380,\n  \"fields\": [\n    {\n      \"column\": \"index\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 67,\n        \"min\": 1,\n        \"max\": 214,\n        \"num_unique_values\": 88,\n        \"samples\": [\n          100,\n          171,\n          9\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"comments\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 378,\n        \"samples\": [\n          \"Alles hat super geklappt Tolle Lage ,saubere Wohnung.\",\n          \"Supersch\\u00f6ne Wohnung, stilvoll und originell mit vielen Antiquit\\u00e4ten und Flair eingerichtet in sehr guter Altstadtlage, sodass alles sehr gut zu erreichen ist! Wir hatten eine tolle Zeit und Pedro stand wirklich jederzeit mit Rat und Flexibilit\\u00e4t zur Verf\\u00fcgung.\",\n          \"Wir hatten eine wundervolle Begr\\u00fc\\u00dfung von Tiago._x000D_<br/>Tiago, der in Ruis Auftrag f\\u00fcr uns da war, hat uns sehr lieb und_x000D_<br/>aufmerksam betreut. Er ist eine Perle._x000D_<br/>Er hat uns auch sehr geholfen, als meinem Mann die Brieftasche _x000D_<br/>gestohlen wurde. Nochmals vielen vielen Dank._x000D_<br/>Als Rat f\\u00fcr jeden Reisenden nach Lissabon. Passt auf Eure Sachen_x000D_<br/>auf, es wird viel gestohlen._x000D_<br/>\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"language\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"de\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment.iloc[0]['language']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1s6n-atITmRx",
        "outputId": "a72b4f6d-7d0e-4123-a742-a9035d3b7b19"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'de'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "translations = []\n",
        "\n",
        "lang = df_experiment.iloc[0]['language']\n",
        "\n",
        "for i in tqdm(range(len(df_experiment))):\n",
        "  if df_experiment['language'].iloc[i] == 'en':\n",
        "    translations.append(df_experiment['comments'].iloc[i])\n",
        "  elif df_experiment['language'].iloc[i] != lang:\n",
        "    lang = df_experiment['language'].iloc[i]\n",
        "    print('\\n New language : ', lang, '\\n')\n",
        "    try:\n",
        "      tokenizer = AutoTokenizer.from_pretrained(f\"Helsinki-NLP/opus-mt-{lang}-en\")\n",
        "      model = AutoModelForSeq2SeqLM.from_pretrained(f\"Helsinki-NLP/opus-mt-{lang}-en\")\n",
        "      pipe = pipeline(\"translation\", model=model, tokenizer=tokenizer, device = 0)\n",
        "      translations.append(pipe(df_experiment['comments'].iloc[i], max_length=10000)[0]['translation_text'])\n",
        "    except:\n",
        "      translations.append('')\n",
        "  else:\n",
        "      translations.append(pipe(df_experiment['comments'].iloc[i], max_length=10000)[0]['translation_text'])\n",
        "      translations.append('')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "id": "v3JTQaWrv-BW",
        "outputId": "1f6ab527-1535-4598-8eb2-05ec87a8aeea"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/380 [00:00<?, ?it/s]Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "  0%|          | 0/380 [00:00<?, ?it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "'translation_text'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-4ec15d919c7d>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m       \u001b[0mtranslations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m       \u001b[0mtranslations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_experiment\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'comments'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'translation_text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m       \u001b[0mtranslations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'translation_text'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pipe(df_experiment['comments'].iloc[i], max_length=10000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ahkTjNEBUwNa",
        "outputId": "ed56ef6d-d398-4e6a-c2cc-97f4ffb2af49"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'de', 'score': 0.9907051920890808}]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translations"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fJ7tjRJDS9YV",
        "outputId": "948631cb-aff1-482b-8cac-5fbb0f013df3"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '']"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment['tr'] = translations"
      ],
      "metadata": {
        "id": "mfRfb_lbSdKF"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment[df_experiment['language'] == 'de']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "8fCLYJW-SzKX",
        "outputId": "61195527-e330-41ab-f8d0-3dfd9a910744"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      index                                           comments language tr\n",
              "8132    171                         Der ideale Ort zum Rekaxen       de   \n",
              "9603    204  Wir sind zum ersten Mal as Familie mit zwei gr...       de   \n",
              "9845    207  Die Wohung von Clement liegt perfekt, nur weni...       de   \n",
              "1569     30  Der Garten und der Pool sind ein Traum. Den Po...       de   \n",
              "5711    119  Gerne wieder beim nächsten Aufenthalt in Lissabon       de   \n",
              "...     ...                                                ...      ... ..\n",
              "9071    194  Tolle Lage und schöne Wohnung. Viel schöne Res...       de   \n",
              "7580    161  Wir haben uns im Castro Beach House wirklich s...       de   \n",
              "6883    146  Die Kommunikation war super alles hat schnell ...       de   \n",
              "5140    100  angenehm und unkompliziert. die wohnung ist ei...       de   \n",
              "1725     35  Alles war perfekt: die Gastgeber, das Haus, di...       de   \n",
              "\n",
              "[380 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-03ff67c7-3dae-4b34-9a6f-a7d134179ece\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>comments</th>\n",
              "      <th>language</th>\n",
              "      <th>tr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8132</th>\n",
              "      <td>171</td>\n",
              "      <td>Der ideale Ort zum Rekaxen</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9603</th>\n",
              "      <td>204</td>\n",
              "      <td>Wir sind zum ersten Mal as Familie mit zwei gr...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9845</th>\n",
              "      <td>207</td>\n",
              "      <td>Die Wohung von Clement liegt perfekt, nur weni...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1569</th>\n",
              "      <td>30</td>\n",
              "      <td>Der Garten und der Pool sind ein Traum. Den Po...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5711</th>\n",
              "      <td>119</td>\n",
              "      <td>Gerne wieder beim nächsten Aufenthalt in Lissabon</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9071</th>\n",
              "      <td>194</td>\n",
              "      <td>Tolle Lage und schöne Wohnung. Viel schöne Res...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7580</th>\n",
              "      <td>161</td>\n",
              "      <td>Wir haben uns im Castro Beach House wirklich s...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6883</th>\n",
              "      <td>146</td>\n",
              "      <td>Die Kommunikation war super alles hat schnell ...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5140</th>\n",
              "      <td>100</td>\n",
              "      <td>angenehm und unkompliziert. die wohnung ist ei...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1725</th>\n",
              "      <td>35</td>\n",
              "      <td>Alles war perfekt: die Gastgeber, das Haus, di...</td>\n",
              "      <td>de</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>380 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-03ff67c7-3dae-4b34-9a6f-a7d134179ece')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-03ff67c7-3dae-4b34-9a6f-a7d134179ece button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-03ff67c7-3dae-4b34-9a6f-a7d134179ece');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ba76be2c-1f38-4420-a0af-f6d8e818d24f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ba76be2c-1f38-4420-a0af-f6d8e818d24f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ba76be2c-1f38-4420-a0af-f6d8e818d24f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "repr_error": "0"
            }
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(translations)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-6F1wduhSnOL",
        "outputId": "322d1208-5a56-4605-8b01-a138dabfe109"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_experiment.tr.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H8nC2dFuSlEx",
        "outputId": "8a5c3c3f-e1d1-4d05-8acb-e632cf0f1ba3"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tr\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                              3647\n",
              ".                                                                                                                                                                                                                                                                                                                                                                                                                                                                               15\n",
              "Great place!                                                                                                                                                                                                                                                                                                                                                                                                                                                                     7\n",
              "Great location!                                                                                                                                                                                                                                                                                                                                                                                                                                                                  5\n",
              "Great                                                                                                                                                                                                                                                                                                                                                                                                                                                                            3\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                              ... \n",
              "Great location, great space, good mattresses, sheets, and pillows, very clean.  If you can climb 5 floors' worth of stairs, this is a great space.                                                                                                                                                                                                                                                                                                                               1\n",
              "The location couldn’t be any better. It’s close to many of the historical attractions, great restaurants and always felt very safe, even late at night. The apartment was super clean, bright and had everything you could need. It is called ‘Se Attic’ for a reason so, if you have any issues with stairs this might not be the place for you. I would most definitely book and stay again. I would recommend this place BIGTIME!! We Loved It!!                              1\n",
              "Sofia’s apartment was wonderful!  It is a truly modern and comfortable apartment. The elevator was a great addition. To be able to see that view from the amazing terrace was magical.  The view throughout the day and night  was incredible!  Sofia was a very clear.communicator and she was accommodating in every way.  The neighborhood is wonderful and central.  There were several hidden gems in the area.  The apartment is perfectly positioned atop the hill.       1\n",
              "Great location with fantastic views and light                                                                                                                                                                                                                                                                                                                                                                                                                                    1\n",
              "Se attic was a fantastic apartment<br/><br/>Very charismatic host<br/><br/>Great communication<br/><br/>Superb, clean apartment<br/><br/>Perfect for our trip                                                                                                                                                                                                                                                                                                                    1\n",
              "Name: count, Length: 6306, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translator = pipeline(\"translation\", model=model, tokenizer=tokenizer)\n",
        "\n",
        "# Example sentence to translate\n",
        "sentence = \"Привет, как дела?\"\n",
        "\n",
        "# Translate the sentence\n",
        "translated_sentence = translator(sentence, max_length=50)[0]['translation_text']\n",
        "\n",
        "# Print the translated sentence\n",
        "print(\"Translated Sentence:\", translated_sentence)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RORqd-BCsz8P",
        "outputId": "9ba4d9ec-4863-457e-84c0-1b087fc8bce1"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Translated Sentence: Hey, how's it going?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "a"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "WFCf4W02PZA5",
        "outputId": "da894baa-57dc-4bb5-b3b3-f3cc8f781708"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Si'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences_russian = [\n",
        "    \"Я люблю гулять по парку вечером.\",\n",
        "    \"Сегодня очень хорошая погода.\",\n",
        "    \"У меня много работы на этой неделе.\",\n",
        "    \"Мой друг приезжает в гости завтра.\",\n",
        "    \"Я хочу посмотреть новый фильм в кино.\",\n",
        "    \"Моя сестра учится в университете.\",\n",
        "    \"Вчера я готовил пасту с морепродуктами.\",\n",
        "    \"Он хочет научиться играть на гитаре.\",\n",
        "    \"Утром всегда пью кофе с молоком.\",\n",
        "    \"Я люблю читать книги на свежем воздухе.\",\n",
        "    \"В субботу планирую посетить музей.\",\n",
        "    \"Мы с друзьями собираемся на пикник на выходных.\",\n",
        "    \"Моя собака очень ласковая и дружелюбная.\",\n",
        "    \"Она хочет поехать в отпуск на море.\",\n",
        "    \"Я мечтаю научиться говорить на английском свободно.\",\n",
        "    \"Сегодня вечером я иду в театр с мамой.\",\n",
        "    \"У нас в саду растут красивые цветы.\",\n",
        "    \"Летом мы часто ездим на дачу.\",\n",
        "    \"Вчера я посетил новую кафе в центре города.\",\n",
        "    \"Он хочет купить новый автомобиль этой осенью.\"\n",
        "]\n",
        "\n",
        "for i in sentences_russian:\n",
        "  translated_sentence = translator(i, max_length=50)[0]['translation_text']\n",
        "\n",
        "  # Print the translated sentence\n",
        "  print(\"Translated Sentence:\", translated_sentence)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bdzc7Sk-tRFd",
        "outputId": "dc3369b3-4eac-4529-81d5-a6ded2703f9b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Translated Sentence: I like to walk in the park tonight.\n",
            "Translated Sentence: It's very good weather today.\n",
            "Translated Sentence: I have a lot of work to do this week.\n",
            "Translated Sentence: My friend's coming over tomorrow.\n",
            "Translated Sentence: I want to see a new movie in the movies.\n",
            "Translated Sentence: My sister goes to university.\n",
            "Translated Sentence: I cooked pasta with seafood yesterday.\n",
            "Translated Sentence: He wants to learn to play the guitar.\n",
            "Translated Sentence: I always drink coffee and milk in the morning.\n",
            "Translated Sentence: I like reading books in the fresh air.\n",
            "Translated Sentence: I'm planning to visit the museum on Saturday.\n",
            "Translated Sentence: My friends and I are going on a picnic this weekend.\n",
            "Translated Sentence: My dog is very sweet and friendly.\n",
            "Translated Sentence: She wants to go on vacation at sea.\n",
            "Translated Sentence: I dream of learning to speak English freely.\n",
            "Translated Sentence: I'm going to the theater with my mom tonight.\n",
            "Translated Sentence: Beautiful flowers grow in our garden.\n",
            "Translated Sentence: We often go to the gym in the summer.\n",
            "Translated Sentence: Yesterday, I visited a new cafe downtown.\n",
            "Translated Sentence: He wants to buy a new car this fall.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GKh1CJpftnKQ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}