Code_for_Inference_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3bab5dff66b8404f80f10cf3da5b1f70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3afff27deef4431eb7b9521da31cb78c",
              "IPY_MODEL_33debfc43a4a45bd921ab249b877c4f2",
              "IPY_MODEL_4c0e536ac61743a99ca5647802e43f8d"
            ],
            "layout": "IPY_MODEL_a0d2e1e88fe345faa018b7d46989ee91"
          }
        },
        "3afff27deef4431eb7b9521da31cb78c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2e1773160a2480c87fb7a6cfe2e057d",
            "placeholder": "​",
            "style": "IPY_MODEL_91bb99cd231f4ba98958b4be9f8a8a3e",
            "value": "Downloading: 100%"
          }
        },
        "33debfc43a4a45bd921ab249b877c4f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95ed41c606034ab9958e41cde35bb76a",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_517831354b7f492b998a440edba80f8e",
            "value": 29
          }
        },
        "4c0e536ac61743a99ca5647802e43f8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0ea0c4d7d154729ac014d3e5ac11f26",
            "placeholder": "​",
            "style": "IPY_MODEL_bc7f1c0efe87402d9eea2683ca82a999",
            "value": " 29.0/29.0 [00:00&lt;00:00, 298B/s]"
          }
        },
        "a0d2e1e88fe345faa018b7d46989ee91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2e1773160a2480c87fb7a6cfe2e057d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91bb99cd231f4ba98958b4be9f8a8a3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "95ed41c606034ab9958e41cde35bb76a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "517831354b7f492b998a440edba80f8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0ea0c4d7d154729ac014d3e5ac11f26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc7f1c0efe87402d9eea2683ca82a999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "17b76c7dc2e24201be0aba057c9e85f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e0b7fb7e5c5b4020a53451ce421cf8eb",
              "IPY_MODEL_7bd016fc0c2a4baa8ade21e7a6302722",
              "IPY_MODEL_519bab74dc404b46a01a4aee87d32e97"
            ],
            "layout": "IPY_MODEL_2650a4120f904daba8c4eb70f1085a6f"
          }
        },
        "e0b7fb7e5c5b4020a53451ce421cf8eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86a7421eddda4b2791445523b86858bd",
            "placeholder": "​",
            "style": "IPY_MODEL_9cd8776b08744bff9e389174ab158523",
            "value": "Downloading: 100%"
          }
        },
        "7bd016fc0c2a4baa8ade21e7a6302722": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fafa899e40ef44a39eeb84949cb5b03b",
            "max": 762,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2cb1c6a95bc34489aeecca42aad03d1c",
            "value": 762
          }
        },
        "519bab74dc404b46a01a4aee87d32e97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89d6df8b2c7f43f0868edc684304744f",
            "placeholder": "​",
            "style": "IPY_MODEL_39725f8990c84839b183da3d08e4aedb",
            "value": " 762/762 [00:00&lt;00:00, 11.0kB/s]"
          }
        },
        "2650a4120f904daba8c4eb70f1085a6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86a7421eddda4b2791445523b86858bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cd8776b08744bff9e389174ab158523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fafa899e40ef44a39eeb84949cb5b03b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cb1c6a95bc34489aeecca42aad03d1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "89d6df8b2c7f43f0868edc684304744f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39725f8990c84839b183da3d08e4aedb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f7ee136c3c74c37a9ff63c4b3810478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c1aeba22e684445a957edb23248f5b03",
              "IPY_MODEL_10ff2ab161a94c55b1ae07a6f3f15874",
              "IPY_MODEL_d2d82e08322b43b389882cf15670144b"
            ],
            "layout": "IPY_MODEL_f8e7346c61b047b3a78a688cfd277585"
          }
        },
        "c1aeba22e684445a957edb23248f5b03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4168e7736aca4f3bab10591df284bd9a",
            "placeholder": "​",
            "style": "IPY_MODEL_f96f3848c8dc4609af6a09ff7176e844",
            "value": "Downloading: 100%"
          }
        },
        "10ff2ab161a94c55b1ae07a6f3f15874": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae4fe9245c144054a0c8eedb61210ea2",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_352fab0d21524589ad3c82db18e1d229",
            "value": 213450
          }
        },
        "d2d82e08322b43b389882cf15670144b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8bde4428dd6446819fa873e9cffb9b25",
            "placeholder": "​",
            "style": "IPY_MODEL_1a38650676d6471aaecbd31e460d7f20",
            "value": " 208k/208k [00:00&lt;00:00, 2.15MB/s]"
          }
        },
        "f8e7346c61b047b3a78a688cfd277585": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4168e7736aca4f3bab10591df284bd9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f96f3848c8dc4609af6a09ff7176e844": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ae4fe9245c144054a0c8eedb61210ea2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "352fab0d21524589ad3c82db18e1d229": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8bde4428dd6446819fa873e9cffb9b25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a38650676d6471aaecbd31e460d7f20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a1a66709f5d4091a01314cb90394d5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d96640f1f9a04ae1b5abf79bd41d52a1",
              "IPY_MODEL_54a3306149a44ab7890412ff22abfd26",
              "IPY_MODEL_a240788c91ab4e3bbe6ee03bf55129c6"
            ],
            "layout": "IPY_MODEL_933c140fc4804aabbc5e0ce5b841c2a6"
          }
        },
        "d96640f1f9a04ae1b5abf79bd41d52a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8f9dd2510954ce4817d53abb15838fc",
            "placeholder": "​",
            "style": "IPY_MODEL_2b1e9c2849df4ae9b4eee9156cb99cca",
            "value": "Downloading: 100%"
          }
        },
        "54a3306149a44ab7890412ff22abfd26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0dd6c5f7e13347beb702f92382378ef3",
            "max": 435797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f44e9b68fc0b4b4b999b852fab321c19",
            "value": 435797
          }
        },
        "a240788c91ab4e3bbe6ee03bf55129c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a09c267af2754fceb078bae7aad7350a",
            "placeholder": "​",
            "style": "IPY_MODEL_16ba6332c1be4fd4822a4443addea713",
            "value": " 426k/426k [00:00&lt;00:00, 1.70MB/s]"
          }
        },
        "933c140fc4804aabbc5e0ce5b841c2a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8f9dd2510954ce4817d53abb15838fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b1e9c2849df4ae9b4eee9156cb99cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0dd6c5f7e13347beb702f92382378ef3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f44e9b68fc0b4b4b999b852fab321c19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a09c267af2754fceb078bae7aad7350a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16ba6332c1be4fd4822a4443addea713": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "106f73744c2a44abbf31094b8ef91a6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b04940e66fb441a7be5c3e48e2f2a0fb",
              "IPY_MODEL_a1f7a396fe0f4c0e83f8bb7163ee9ead",
              "IPY_MODEL_7d0a0392d8d84e7bb9ddebaad461bcff"
            ],
            "layout": "IPY_MODEL_02e4ee67dd804a70b089e51da01e8abf"
          }
        },
        "b04940e66fb441a7be5c3e48e2f2a0fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9cf61e7336f6437fa00906b88cd8f494",
            "placeholder": "​",
            "style": "IPY_MODEL_b6e310e8f4b44e0c8e4d2b1997c62f29",
            "value": "Downloading: 100%"
          }
        },
        "a1f7a396fe0f4c0e83f8bb7163ee9ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62a9a808287c497a80fc522eecda1243",
            "max": 1338740706,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_714ccb5b18534f099b40f1a1f7cb1d3d",
            "value": 1338740706
          }
        },
        "7d0a0392d8d84e7bb9ddebaad461bcff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_150c7930a99f4f6aa074a42ce45e5f20",
            "placeholder": "​",
            "style": "IPY_MODEL_2d39f2622fa04ef784454f656069da1e",
            "value": " 1.25G/1.25G [00:41&lt;00:00, 54.2MB/s]"
          }
        },
        "02e4ee67dd804a70b089e51da01e8abf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cf61e7336f6437fa00906b88cd8f494": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6e310e8f4b44e0c8e4d2b1997c62f29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "62a9a808287c497a80fc522eecda1243": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "714ccb5b18534f099b40f1a1f7cb1d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "150c7930a99f4f6aa074a42ce45e5f20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d39f2622fa04ef784454f656069da1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7N-A7Bziq21",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2392cacd-21f1-4fc6-90fb-2a59fa789b50"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install textdistance"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.20.1-py3-none-any.whl (4.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 5.1 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.8.1-py3-none-any.whl (101 kB)\n",
            "\u001b[K     |████████████████████████████████| 101 kB 12.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.12.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.1)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 65.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 47.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.8.1 pyyaml-6.0 tokenizers-0.12.1 transformers-4.20.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting textdistance\n",
            "  Downloading textdistance-4.3.0-py3-none-any.whl (29 kB)\n",
            "Installing collected packages: textdistance\n",
            "Successfully installed textdistance-4.3.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8SQs7G5Wh-4L",
        "outputId": "5a3802f0-7c94-48d2-bd05-063b0168c8c9"
      },
      "source": [
        "import pandas as pd\n",
        "import nltk\n",
        "import os\n",
        "from random import shuffle\n",
        "from random import randint\n",
        "import re\n",
        "import textdistance\n",
        "nltk.download('punkt')\n",
        "\n",
        "from transformers import AutoModelWithLMHead, AutoTokenizer\n",
        "import torch\n",
        "from transformers import AutoConfig, AutoModelForMaskedLM\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289,
          "referenced_widgets": [
            "3bab5dff66b8404f80f10cf3da5b1f70",
            "3afff27deef4431eb7b9521da31cb78c",
            "33debfc43a4a45bd921ab249b877c4f2",
            "4c0e536ac61743a99ca5647802e43f8d",
            "a0d2e1e88fe345faa018b7d46989ee91",
            "c2e1773160a2480c87fb7a6cfe2e057d",
            "91bb99cd231f4ba98958b4be9f8a8a3e",
            "95ed41c606034ab9958e41cde35bb76a",
            "517831354b7f492b998a440edba80f8e",
            "b0ea0c4d7d154729ac014d3e5ac11f26",
            "bc7f1c0efe87402d9eea2683ca82a999",
            "17b76c7dc2e24201be0aba057c9e85f2",
            "e0b7fb7e5c5b4020a53451ce421cf8eb",
            "7bd016fc0c2a4baa8ade21e7a6302722",
            "519bab74dc404b46a01a4aee87d32e97",
            "2650a4120f904daba8c4eb70f1085a6f",
            "86a7421eddda4b2791445523b86858bd",
            "9cd8776b08744bff9e389174ab158523",
            "fafa899e40ef44a39eeb84949cb5b03b",
            "2cb1c6a95bc34489aeecca42aad03d1c",
            "89d6df8b2c7f43f0868edc684304744f",
            "39725f8990c84839b183da3d08e4aedb",
            "8f7ee136c3c74c37a9ff63c4b3810478",
            "c1aeba22e684445a957edb23248f5b03",
            "10ff2ab161a94c55b1ae07a6f3f15874",
            "d2d82e08322b43b389882cf15670144b",
            "f8e7346c61b047b3a78a688cfd277585",
            "4168e7736aca4f3bab10591df284bd9a",
            "f96f3848c8dc4609af6a09ff7176e844",
            "ae4fe9245c144054a0c8eedb61210ea2",
            "352fab0d21524589ad3c82db18e1d229",
            "8bde4428dd6446819fa873e9cffb9b25",
            "1a38650676d6471aaecbd31e460d7f20",
            "5a1a66709f5d4091a01314cb90394d5e",
            "d96640f1f9a04ae1b5abf79bd41d52a1",
            "54a3306149a44ab7890412ff22abfd26",
            "a240788c91ab4e3bbe6ee03bf55129c6",
            "933c140fc4804aabbc5e0ce5b841c2a6",
            "e8f9dd2510954ce4817d53abb15838fc",
            "2b1e9c2849df4ae9b4eee9156cb99cca",
            "0dd6c5f7e13347beb702f92382378ef3",
            "f44e9b68fc0b4b4b999b852fab321c19",
            "a09c267af2754fceb078bae7aad7350a",
            "16ba6332c1be4fd4822a4443addea713",
            "106f73744c2a44abbf31094b8ef91a6e",
            "b04940e66fb441a7be5c3e48e2f2a0fb",
            "a1f7a396fe0f4c0e83f8bb7163ee9ead",
            "7d0a0392d8d84e7bb9ddebaad461bcff",
            "02e4ee67dd804a70b089e51da01e8abf",
            "9cf61e7336f6437fa00906b88cd8f494",
            "b6e310e8f4b44e0c8e4d2b1997c62f29",
            "62a9a808287c497a80fc522eecda1243",
            "714ccb5b18534f099b40f1a1f7cb1d3d",
            "150c7930a99f4f6aa074a42ce45e5f20",
            "2d39f2622fa04ef784454f656069da1e"
          ]
        },
        "id": "WzvefoIWiAx2",
        "outputId": "bcc5bdcf-e937-43b6-8bc0-c6c0428e0f7a"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-large-cased\")\n",
        "model = AutoModelWithLMHead.from_pretrained(\"bert-large-cased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3bab5dff66b8404f80f10cf3da5b1f70"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/762 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "17b76c7dc2e24201be0aba057c9e85f2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/208k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8f7ee136c3c74c37a9ff63c4b3810478"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/426k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a1a66709f5d4091a01314cb90394d5e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/models/auto/modeling_auto.py:973: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.25G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "106f73744c2a44abbf31094b8ef91a6e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-large-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lGkYN51LjOhg"
      },
      "source": [
        "#reading the csv file containing the video id, the slur, the matching excerpts from the youtube and amazon transcripts, and the ground truth for the slur\n",
        "df=pd.read_csv('CSV/Dataset_Amazon_Final_2.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 676
        },
        "id": "B3Vm9fPUH9_s",
        "outputId": "b7a96820-b527-4c6f-b0d6-b5aad5bab44c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        Id      Slur                                            YouTube  \\\n",
              "0       53       ass  ready for a rescue shes going to take to the s...   \n",
              "1      218     bitch                            his life hey bitch mean   \n",
              "2      312  cocktail  also exceptionally strong for their size they ...   \n",
              "3      326      crap                        the red crap is standing on   \n",
              "4      360      crap  stretchy and sticky and now we have a crap and...   \n",
              "...    ...       ...                                                ...   \n",
              "1654  2110      shit  just for fun climb and ten nine ten dont you w...   \n",
              "1655  1862      rape  spread on the door five and six take your pick...   \n",
              "1656  2123      shit  spread on the door five and six take your pick...   \n",
              "1657   608      dope  boss i dont know who that is guys do you guys ...   \n",
              "1658   834      fuck  school bus song Bring it ♪ The wheels on the b...   \n",
              "\n",
              "                                                 Amazon  Levenshtein  \\\n",
              "0     ready for a rescue shes going to take to the s...            0   \n",
              "1                               his life hey bitch mean            0   \n",
              "2     also exceptionally strong for their size they ...            0   \n",
              "3                           the red crap is standing on            0   \n",
              "4     stretchy and sticky and now we have a crap and...            0   \n",
              "...                                                 ...          ...   \n",
              "1654  just for fun fine and 10 9 and 10 dont you wan...         2643   \n",
              "1655  spread on the door five and six take your pick...         3246   \n",
              "1656  spread on the door five and six take your pick...         3246   \n",
              "1657  boss yeah i dont know that is you guys you guy...         4331   \n",
              "1658  school by san you want yeah mhm we have story ...         5549   \n",
              "\n",
              "                                                   Link  \\\n",
              "0     https://www.youtube.com/watch?v=0Z9rIS4xNZ0&t=...   \n",
              "1     https://www.youtube.com/watch?v=9R4lLcm5p7I&t=...   \n",
              "2     https://www.youtube.com/watch?v=Fw1_1_yKZ78&t=...   \n",
              "3     https://www.youtube.com/watch?v=pmQTB3ScUo8&t=...   \n",
              "4     https://www.youtube.com/watch?v=jrZwy-ybyfA&t=...   \n",
              "...                                                 ...   \n",
              "1654  https://www.youtube.com/watch?v=zwibXy4CCgo&t=...   \n",
              "1655  https://www.youtube.com/watch?v=JZHJ2679XMc&t=...   \n",
              "1656  https://www.youtube.com/watch?v=JZHJ2679XMc&t=...   \n",
              "1657  https://www.youtube.com/watch?v=PKQ6B7r2TrI&t=...   \n",
              "1658  https://www.youtube.com/watch?v=Rzc1exd4iiM&t=...   \n",
              "\n",
              "     Does the audio contain a slur or not?  \\\n",
              "0                                       No   \n",
              "1                                      NaN   \n",
              "2                                       No   \n",
              "3                                       No   \n",
              "4                                       No   \n",
              "...                                    ...   \n",
              "1654                                   NaN   \n",
              "1655                                   NaN   \n",
              "1656                                   NaN   \n",
              "1657                                   NaN   \n",
              "1658                                   NaN   \n",
              "\n",
              "                                 Original Transcription  \\\n",
              "0     ready for a rescue shes going to take to the s...   \n",
              "1                                                   NaN   \n",
              "2     also exceptionally strong for their size they ...   \n",
              "3                           The red crab is standing on   \n",
              "4     stretchy and sticky and now we have a crab and...   \n",
              "...                                                 ...   \n",
              "1654                                                NaN   \n",
              "1655                                                NaN   \n",
              "1656                                                NaN   \n",
              "1657                                                NaN   \n",
              "1658                                                NaN   \n",
              "\n",
              "     Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)  \\\n",
              "0                                       Caucasian Woman                                  \n",
              "1                                                   NaN                                  \n",
              "2                                                   NaN                                  \n",
              "3                                                   NaN                                  \n",
              "4                                                   NaN                                  \n",
              "...                                                 ...                                  \n",
              "1654                                                NaN                                  \n",
              "1655                                                NaN                                  \n",
              "1656                                                NaN                                  \n",
              "1657                                                NaN                                  \n",
              "1658                                                NaN                                  \n",
              "\n",
              "     Actual Word  \n",
              "0           maya  \n",
              "1            NaN  \n",
              "2         copter  \n",
              "3           crab  \n",
              "4           crab  \n",
              "...          ...  \n",
              "1654         NaN  \n",
              "1655         NaN  \n",
              "1656         NaN  \n",
              "1657         NaN  \n",
              "1658         NaN  \n",
              "\n",
              "[1659 rows x 10 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5f9f2b84-e843-4d9d-b910-c01318c60d17\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>Slur</th>\n",
              "      <th>YouTube</th>\n",
              "      <th>Amazon</th>\n",
              "      <th>Levenshtein</th>\n",
              "      <th>Link</th>\n",
              "      <th>Does the audio contain a slur or not?</th>\n",
              "      <th>Original Transcription</th>\n",
              "      <th>Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)</th>\n",
              "      <th>Actual Word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>53</td>\n",
              "      <td>ass</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=0Z9rIS4xNZ0&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>Caucasian Woman</td>\n",
              "      <td>maya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>218</td>\n",
              "      <td>bitch</td>\n",
              "      <td>his life hey bitch mean</td>\n",
              "      <td>his life hey bitch mean</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=9R4lLcm5p7I&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>312</td>\n",
              "      <td>cocktail</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=Fw1_1_yKZ78&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>copter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>326</td>\n",
              "      <td>crap</td>\n",
              "      <td>the red crap is standing on</td>\n",
              "      <td>the red crap is standing on</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=pmQTB3ScUo8&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>The red crab is standing on</td>\n",
              "      <td>NaN</td>\n",
              "      <td>crab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>360</td>\n",
              "      <td>crap</td>\n",
              "      <td>stretchy and sticky and now we have a crap and...</td>\n",
              "      <td>stretchy and sticky and now we have a crap and...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=jrZwy-ybyfA&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>stretchy and sticky and now we have a crab and...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>crab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1654</th>\n",
              "      <td>2110</td>\n",
              "      <td>shit</td>\n",
              "      <td>just for fun climb and ten nine ten dont you w...</td>\n",
              "      <td>just for fun fine and 10 9 and 10 dont you wan...</td>\n",
              "      <td>2643</td>\n",
              "      <td>https://www.youtube.com/watch?v=zwibXy4CCgo&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1655</th>\n",
              "      <td>1862</td>\n",
              "      <td>rape</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>3246</td>\n",
              "      <td>https://www.youtube.com/watch?v=JZHJ2679XMc&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1656</th>\n",
              "      <td>2123</td>\n",
              "      <td>shit</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>3246</td>\n",
              "      <td>https://www.youtube.com/watch?v=JZHJ2679XMc&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1657</th>\n",
              "      <td>608</td>\n",
              "      <td>dope</td>\n",
              "      <td>boss i dont know who that is guys do you guys ...</td>\n",
              "      <td>boss yeah i dont know that is you guys you guy...</td>\n",
              "      <td>4331</td>\n",
              "      <td>https://www.youtube.com/watch?v=PKQ6B7r2TrI&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1658</th>\n",
              "      <td>834</td>\n",
              "      <td>fuck</td>\n",
              "      <td>school bus song Bring it ♪ The wheels on the b...</td>\n",
              "      <td>school by san you want yeah mhm we have story ...</td>\n",
              "      <td>5549</td>\n",
              "      <td>https://www.youtube.com/watch?v=Rzc1exd4iiM&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1659 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5f9f2b84-e843-4d9d-b910-c01318c60d17')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5f9f2b84-e843-4d9d-b910-c01318c60d17 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5f9f2b84-e843-4d9d-b910-c01318c60d17');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[df['Slur'] == \"anus\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        },
        "id": "33QT94mXeHBT",
        "outputId": "0bdd8c2a-4df8-49c3-80ec-baa7d9a26d95"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Id  Slur                                            YouTube  \\\n",
              "485   0  anus  has in hurry nice now we are ready to put our ...   \n",
              "\n",
              "                                                Amazon  Levenshtein  \\\n",
              "485  has in her anus now we are ready to put our do...           14   \n",
              "\n",
              "                                                  Link  \\\n",
              "485  https://www.youtube.com/watch?v=t9rMyBQlvZc&t=...   \n",
              "\n",
              "    Does the audio contain a slur or not? Original Transcription  \\\n",
              "485                                   NaN                    NaN   \n",
              "\n",
              "    Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)  \\\n",
              "485                                                NaN                                  \n",
              "\n",
              "    Actual Word  \n",
              "485         NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5246318e-5c1c-41f3-a757-b2d42e39dff1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>Slur</th>\n",
              "      <th>YouTube</th>\n",
              "      <th>Amazon</th>\n",
              "      <th>Levenshtein</th>\n",
              "      <th>Link</th>\n",
              "      <th>Does the audio contain a slur or not?</th>\n",
              "      <th>Original Transcription</th>\n",
              "      <th>Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)</th>\n",
              "      <th>Actual Word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>485</th>\n",
              "      <td>0</td>\n",
              "      <td>anus</td>\n",
              "      <td>has in hurry nice now we are ready to put our ...</td>\n",
              "      <td>has in her anus now we are ready to put our do...</td>\n",
              "      <td>14</td>\n",
              "      <td>https://www.youtube.com/watch?v=t9rMyBQlvZc&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5246318e-5c1c-41f3-a757-b2d42e39dff1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5246318e-5c1c-41f3-a757-b2d42e39dff1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5246318e-5c1c-41f3-a757-b2d42e39dff1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "list(df[df['Slur'] == \"anus\"]['YouTube']), list(df[df['Slur'] == \"anus\"]['Amazon'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mlnr7LwckOpY",
        "outputId": "9331a2ed-623d-42d4-b52a-9fae808a018e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['has in hurry nice now we are ready to put our doll together'],\n",
              " ['has in her anus now we are ready to put our doll together'])"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"/content/slurs.txt\", \"w\") as file:\n",
        "    for s in list(set(df['Slur'])):\n",
        "        file.write(\"%s\\n\"%s)"
      ],
      "metadata": {
        "id": "Yy2n9GTh_RNV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j8Uu6XvdIdsh",
        "outputId": "397a118d-b37c-491a-ed86-b2634e7daffe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PreTrainedTokenizerFast(name_or_path='bert-large-cased', vocab_size=28996, model_max_len=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V747IEV9iHgq"
      },
      "source": [
        "#constructing a vocabulary\n",
        "vocabulary=[]\n",
        "for i in range(len(tokenizer)):\n",
        "  vocabulary.append(tokenizer.decode([i]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(vocabulary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gzyxz3dzIWpP",
        "outputId": "bb79a802-69ee-4cea-e585-81ab1a7aec6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28996"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ev7gruWiiJbr"
      },
      "source": [
        "#constructs a set of unique words from the given string\n",
        "def unique_words(x):\n",
        "  temp=[]\n",
        "  for i in x:\n",
        "    if i.lower() not in temp:\n",
        "      temp.append(i.lower())\n",
        "  return temp\n",
        "\n",
        "#proposes a set of candidate words based on a condition (here, the Levenshtein distance is <=1) and the most likely words as proposed by the MLM\n",
        "def candidate_set(word_, mask_logits):\n",
        "  candidate_scores={}\n",
        "  word_limit, prim_cand=int(len(word_)/2), []\n",
        "\n",
        "  for item in vocabulary:\n",
        "    lev=textdistance.levenshtein(word_, item)\n",
        "    if(lev<=word_limit):\n",
        "      id=vocabulary.index(item)\n",
        "      candidate_scores[item]=mask_logits[0, id].item()\n",
        "    if(lev<=1):\n",
        "      prim_cand.append(item)\n",
        "\n",
        "  return candidate_scores, prim_cand\n",
        "\n",
        "#returns a list of the N most likely candidates, where N is specified by the user\n",
        "def top_cand(dict_, top_tokens):\n",
        "    top_candidates=list(dict_.keys())\n",
        "\n",
        "    if(len(top_candidates)<top_tokens):\n",
        "      top_tokens=len(top_candidates)\n",
        "\n",
        "    return [top_candidates[i] for i in range(top_tokens)]\n",
        "    #return [tokenizer.decode(top_candidates[i]) for i in range(top_tokens)]\n",
        "\n",
        "\n",
        "def remove_from_dict(diction, video_id, prim_cand):\n",
        "  to_be_removed=[]\n",
        "  matching_list=dict_matched_unique_words[video_id]\n",
        "  #print(matching_list)\n",
        "  for i in diction.keys():\n",
        "    if((i not in matching_list) and (i not in prim_cand)):\n",
        "      to_be_removed.append(i)\n",
        "\n",
        "  for i in to_be_removed:\n",
        "    diction.pop(i)\n",
        "\n",
        "  return diction\n",
        "\n",
        "#returns the probability\n",
        "def predict_prob(word_, mask_logits):\n",
        "      predict_token=tokenizer.encode(word_)\n",
        "      predict_token=predict_token[1]\n",
        "\n",
        "      return mask_logits[0, predict_token].item()\n",
        "\n",
        "#returning the list of top N candidates\n",
        "def find_top(diction):\n",
        "  keys_=list(diction.keys())\n",
        "  values_=list(diction.values())\n",
        "  w1,v1=[], []\n",
        "\n",
        "  try:\n",
        "    for x in range(topk):\n",
        "      w1.append(keys_[x])\n",
        "      v1.append(values_[x])\n",
        "  except:\n",
        "    return (w1, v1)\n",
        "\n",
        "  return (w1, v1)\n",
        "\n",
        "#sorting a dictionary by descending order based on its value\n",
        "def sort_di(dict_name):\n",
        "  keylist={k: v for k, v in sorted(dict_name.items(), key=lambda item: item[1], reverse=True)}\n",
        "  return keylist"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TO4UwyEVjHq4",
        "outputId": "601caf26-0274-4d3d-cc0c-606d39feb138"
      },
      "source": [
        "#Getting all the unique words for each transcript and saving them into a dictionary\n",
        "matched=pd.read_csv(\"CSV/Matched Transcripts.csv\")\n",
        "act=df['Actual Word'].to_list()\n",
        "act=[i for i in act if i==i]\n",
        "act=unique_words(act)\n",
        "print(act)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['maya', 'copter', 'crab', 'craft', 'grumpy', 'adult', 'race is', 'version', 'ask', 'dinner', 'done', 'trunk', 'duck', 'heroine', 'thing', 'pink', 'rake', 'shirt', 'sit', 'ah', 'beach', 'benches', 'cut', 'trash', 'dent', 'full', 'fun', 'bugs', 'pose', 'rain', 'shed', \"she's\", 'fish', 'oops', 'vlogger', 'dough', 'draw', 'draft', 'fox', 'look', 'looking', 'kells', 'venus', 'pew', 'sacks', 'sug', 'sleigh', 's', 'adds', 'orbit shh', 'bin', 'bet', 'pitch', 'bootsie', 'birdy', 'cockadoodledo', 'toyitis', 'digly', 'drum', 'job', 'strong', 'box', 'popped', 'pumpkins', 'getting', 'little', 'lick of', 'neck-to-neck', 'pets', 'rainbow corn', 'rubbed', 'set', 'said', 'kinder egg', 'as', 'as is', 'djibouti', 'playdoh', 'door', 'draculaura', 'round cut', 'bag', 'pluck me', 'kindi', 'barbie doll oh', 'moana', 'nuggets', 'to test', 'for more', 'peel', 'rainbow', 'race', 'cents', 'socks', 'ship', 'firm', 'whoosh', 'last', 'us', 'a blue', \"grandma's\", 'd meal', \"digly's\", 'bucket', 'buckle', 'okay', 'the thing', 'black and', 'is', 'frustrating', 'neck', 'junction', 'sheesh', 'check', '[noise]', 'spooky', 'kitty', 'orbit', 'caulk', 'come', 'quandong', 'truck', 'dracalaura', 'fork in the', 'feeling', 'tidmouth', 'pick', 'bowl', 'hug', 'caught', 'grouch', 'crowd', 'grampy', 'jack', 'drums', 'drop', 'bug', 'book', 'pokemon', 'fun yay', 'size look at', 'buccaneer', 'such a', 'sesame', 'challenge', 'pansies', 'shell', 'yet', 'shine', 'kitties', '[noises]', 'squish deelish', 'castle', 'bullet train', 'crack', 'that gives', 'molecules', 'six', 'magic this', 'dense', 'could be', 'that', 'master peck', 'orbeez', 'legs', 'accomplished', \"it's you uh\", 'itchy', 'dig', 'music', 'pup', 'falcon', 'yucky', 'look like a', 'cute', 'they can', 'suspects', 'looks']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(act)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IYq02klDJNP-",
        "outputId": "d51fb003-264e-48bd-de72-1e6382a6feac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "178"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 676
        },
        "id": "9C1e77gEhn9D",
        "outputId": "c4c9c99c-0b95-4dcd-e19e-f4b135e0e9f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        Id      Slur                                            YouTube  \\\n",
              "0       53       ass  ready for a rescue shes going to take to the s...   \n",
              "1      218     bitch                            his life hey bitch mean   \n",
              "2      312  cocktail  also exceptionally strong for their size they ...   \n",
              "3      326      crap                        the red crap is standing on   \n",
              "4      360      crap  stretchy and sticky and now we have a crap and...   \n",
              "...    ...       ...                                                ...   \n",
              "1654  2110      shit  just for fun climb and ten nine ten dont you w...   \n",
              "1655  1862      rape  spread on the door five and six take your pick...   \n",
              "1656  2123      shit  spread on the door five and six take your pick...   \n",
              "1657   608      dope  boss i dont know who that is guys do you guys ...   \n",
              "1658   834      fuck  school bus song Bring it ♪ The wheels on the b...   \n",
              "\n",
              "                                                 Amazon  Levenshtein  \\\n",
              "0     ready for a rescue shes going to take to the s...            0   \n",
              "1                               his life hey bitch mean            0   \n",
              "2     also exceptionally strong for their size they ...            0   \n",
              "3                           the red crap is standing on            0   \n",
              "4     stretchy and sticky and now we have a crap and...            0   \n",
              "...                                                 ...          ...   \n",
              "1654  just for fun fine and 10 9 and 10 dont you wan...         2643   \n",
              "1655  spread on the door five and six take your pick...         3246   \n",
              "1656  spread on the door five and six take your pick...         3246   \n",
              "1657  boss yeah i dont know that is you guys you guy...         4331   \n",
              "1658  school by san you want yeah mhm we have story ...         5549   \n",
              "\n",
              "                                                   Link  \\\n",
              "0     https://www.youtube.com/watch?v=0Z9rIS4xNZ0&t=...   \n",
              "1     https://www.youtube.com/watch?v=9R4lLcm5p7I&t=...   \n",
              "2     https://www.youtube.com/watch?v=Fw1_1_yKZ78&t=...   \n",
              "3     https://www.youtube.com/watch?v=pmQTB3ScUo8&t=...   \n",
              "4     https://www.youtube.com/watch?v=jrZwy-ybyfA&t=...   \n",
              "...                                                 ...   \n",
              "1654  https://www.youtube.com/watch?v=zwibXy4CCgo&t=...   \n",
              "1655  https://www.youtube.com/watch?v=JZHJ2679XMc&t=...   \n",
              "1656  https://www.youtube.com/watch?v=JZHJ2679XMc&t=...   \n",
              "1657  https://www.youtube.com/watch?v=PKQ6B7r2TrI&t=...   \n",
              "1658  https://www.youtube.com/watch?v=Rzc1exd4iiM&t=...   \n",
              "\n",
              "     Does the audio contain a slur or not?  \\\n",
              "0                                       No   \n",
              "1                                      NaN   \n",
              "2                                       No   \n",
              "3                                       No   \n",
              "4                                       No   \n",
              "...                                    ...   \n",
              "1654                                   NaN   \n",
              "1655                                   NaN   \n",
              "1656                                   NaN   \n",
              "1657                                   NaN   \n",
              "1658                                   NaN   \n",
              "\n",
              "                                 Original Transcription  \\\n",
              "0     ready for a rescue shes going to take to the s...   \n",
              "1                                                   NaN   \n",
              "2     also exceptionally strong for their size they ...   \n",
              "3                           The red crab is standing on   \n",
              "4     stretchy and sticky and now we have a crab and...   \n",
              "...                                                 ...   \n",
              "1654                                                NaN   \n",
              "1655                                                NaN   \n",
              "1656                                                NaN   \n",
              "1657                                                NaN   \n",
              "1658                                                NaN   \n",
              "\n",
              "     Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)  \\\n",
              "0                                       Caucasian Woman                                  \n",
              "1                                                   NaN                                  \n",
              "2                                                   NaN                                  \n",
              "3                                                   NaN                                  \n",
              "4                                                   NaN                                  \n",
              "...                                                 ...                                  \n",
              "1654                                                NaN                                  \n",
              "1655                                                NaN                                  \n",
              "1656                                                NaN                                  \n",
              "1657                                                NaN                                  \n",
              "1658                                                NaN                                  \n",
              "\n",
              "     Actual Word  \n",
              "0           maya  \n",
              "1            NaN  \n",
              "2         copter  \n",
              "3           crab  \n",
              "4           crab  \n",
              "...          ...  \n",
              "1654         NaN  \n",
              "1655         NaN  \n",
              "1656         NaN  \n",
              "1657         NaN  \n",
              "1658         NaN  \n",
              "\n",
              "[1659 rows x 10 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1969c370-a080-41ee-9d03-4e79c37e66c2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>Slur</th>\n",
              "      <th>YouTube</th>\n",
              "      <th>Amazon</th>\n",
              "      <th>Levenshtein</th>\n",
              "      <th>Link</th>\n",
              "      <th>Does the audio contain a slur or not?</th>\n",
              "      <th>Original Transcription</th>\n",
              "      <th>Race of speaker (Caucasian, Asian, South Asian, Black, and Australoid/ Animation)</th>\n",
              "      <th>Actual Word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>53</td>\n",
              "      <td>ass</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=0Z9rIS4xNZ0&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>ready for a rescue shes going to take to the s...</td>\n",
              "      <td>Caucasian Woman</td>\n",
              "      <td>maya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>218</td>\n",
              "      <td>bitch</td>\n",
              "      <td>his life hey bitch mean</td>\n",
              "      <td>his life hey bitch mean</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=9R4lLcm5p7I&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>312</td>\n",
              "      <td>cocktail</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=Fw1_1_yKZ78&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>also exceptionally strong for their size they ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>copter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>326</td>\n",
              "      <td>crap</td>\n",
              "      <td>the red crap is standing on</td>\n",
              "      <td>the red crap is standing on</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=pmQTB3ScUo8&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>The red crab is standing on</td>\n",
              "      <td>NaN</td>\n",
              "      <td>crab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>360</td>\n",
              "      <td>crap</td>\n",
              "      <td>stretchy and sticky and now we have a crap and...</td>\n",
              "      <td>stretchy and sticky and now we have a crap and...</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.youtube.com/watch?v=jrZwy-ybyfA&amp;t=...</td>\n",
              "      <td>No</td>\n",
              "      <td>stretchy and sticky and now we have a crab and...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>crab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1654</th>\n",
              "      <td>2110</td>\n",
              "      <td>shit</td>\n",
              "      <td>just for fun climb and ten nine ten dont you w...</td>\n",
              "      <td>just for fun fine and 10 9 and 10 dont you wan...</td>\n",
              "      <td>2643</td>\n",
              "      <td>https://www.youtube.com/watch?v=zwibXy4CCgo&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1655</th>\n",
              "      <td>1862</td>\n",
              "      <td>rape</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>3246</td>\n",
              "      <td>https://www.youtube.com/watch?v=JZHJ2679XMc&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1656</th>\n",
              "      <td>2123</td>\n",
              "      <td>shit</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>spread on the door five and six take your pick...</td>\n",
              "      <td>3246</td>\n",
              "      <td>https://www.youtube.com/watch?v=JZHJ2679XMc&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1657</th>\n",
              "      <td>608</td>\n",
              "      <td>dope</td>\n",
              "      <td>boss i dont know who that is guys do you guys ...</td>\n",
              "      <td>boss yeah i dont know that is you guys you guy...</td>\n",
              "      <td>4331</td>\n",
              "      <td>https://www.youtube.com/watch?v=PKQ6B7r2TrI&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1658</th>\n",
              "      <td>834</td>\n",
              "      <td>fuck</td>\n",
              "      <td>school bus song Bring it ♪ The wheels on the b...</td>\n",
              "      <td>school by san you want yeah mhm we have story ...</td>\n",
              "      <td>5549</td>\n",
              "      <td>https://www.youtube.com/watch?v=Rzc1exd4iiM&amp;t=...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1659 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1969c370-a080-41ee-9d03-4e79c37e66c2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1969c370-a080-41ee-9d03-4e79c37e66c2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1969c370-a080-41ee-9d03-4e79c37e66c2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "matched.head(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "3wl81f1dKBvQ",
        "outputId": "3079ec06-60e8-41ad-d2b4-31c590a26d85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0     Video Id                                             Amazon  \\\n",
              "0           0  eSFxZzyH5nM  Oh hey thank you Thank you Welcome to open sta...   \n",
              "1           1  M4YSHQv0pqM  I scream too much sugar What That's not just o...   \n",
              "2           2  1-3xYUwNxeQ  I don't remember how he looks like What does r...   \n",
              "\n",
              "                                             YouTube  \\\n",
              "0  HOOTS: Oh! AUDIENCE: [APPLAUSE] HOOTS: Hey you...   \n",
              "1  [Music] [Music] their days so we feed him some...   \n",
              "2  I don't remember how he looked like what does ...   \n",
              "\n",
              "                                  Matched Transcript  Match Score  \n",
              "0  hey thank welcome stage at everyone can step u...     0.518750  \n",
              "1  you were play doesnt spit comes here put face ...     0.289474  \n",
              "2  remember how he does youre original welcome hi...     0.495614  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8f263441-9574-4048-b5c9-29453a2d6333\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Video Id</th>\n",
              "      <th>Amazon</th>\n",
              "      <th>YouTube</th>\n",
              "      <th>Matched Transcript</th>\n",
              "      <th>Match Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>eSFxZzyH5nM</td>\n",
              "      <td>Oh hey thank you Thank you Welcome to open sta...</td>\n",
              "      <td>HOOTS: Oh! AUDIENCE: [APPLAUSE] HOOTS: Hey you...</td>\n",
              "      <td>hey thank welcome stage at everyone can step u...</td>\n",
              "      <td>0.518750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>M4YSHQv0pqM</td>\n",
              "      <td>I scream too much sugar What That's not just o...</td>\n",
              "      <td>[Music] [Music] their days so we feed him some...</td>\n",
              "      <td>you were play doesnt spit comes here put face ...</td>\n",
              "      <td>0.289474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>1-3xYUwNxeQ</td>\n",
              "      <td>I don't remember how he looks like What does r...</td>\n",
              "      <td>I don't remember how he looked like what does ...</td>\n",
              "      <td>remember how he does youre original welcome hi...</td>\n",
              "      <td>0.495614</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8f263441-9574-4048-b5c9-29453a2d6333')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8f263441-9574-4048-b5c9-29453a2d6333 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8f263441-9574-4048-b5c9-29453a2d6333');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e0FQjXUjiWMp"
      },
      "source": [
        "#creating a list of words in the Amazon and YouTube dictiona\n",
        "dict_matched={}\n",
        "dict_matched_unique_words={}\n",
        "for i in range(len(matched)):\n",
        "  #dict_matched[matched['Video Id'][i]]=(matched['Amazon'][i], matched['YouTube'][i])\n",
        "  words_in_transcript=(str(matched['Amazon'][i]) + str(matched['YouTube'][i])).lower().split()\n",
        "  words_in_transcript=unique_words(words_in_transcript)\n",
        "  for item in act:\n",
        "    if(item+'s') in words_in_transcript and item not in words_in_transcript:\n",
        "      words_in_transcript.append(item)\n",
        "  dict_matched_unique_words[matched['Video Id'][i]]=words_in_transcript"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#dict_matched_unique_words['18tQBHqsw-A']"
      ],
      "metadata": {
        "id": "itlta9GULg2-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-uyDFTGXiL2X"
      },
      "source": [
        "#Masking the words and predicting the r\n",
        "def print_results(df, type_, top_k_words):\n",
        "    #the masked token will vary with every model\n",
        "    repl_word =\" [MASK] \"\n",
        "    count=0\n",
        "    n_count=0\n",
        "    cand_count=0\n",
        "\n",
        "    for i in range(len(df)):\n",
        "      try:\n",
        "\n",
        "          actual_word=df['Actual Word'][i]\n",
        "          if((df['Does the audio contain a slur or not?'][i]!='No') or (actual_word=='[Noises]')):\n",
        "            continue\n",
        "          am=str(df[type_][i])\n",
        "          word=str(df['Slur'][i])\n",
        "          vid_id=str(df['Link'][i])[32:43]\n",
        "\n",
        "          #replaces the word with the masked token\n",
        "          res = am.replace(word, repl_word, 1)\n",
        "          input = tokenizer.encode(res, return_tensors=\"pt\")\n",
        "          mask_token_index = torch.where(input == tokenizer.mask_token_id)[1]\n",
        "          #predicts the masked token\n",
        "          token_logits = model(input).logits\n",
        "          mask_token_logits = token_logits[0, mask_token_index, :]\n",
        "\n",
        "          #print(\"Amazon:\", res)\n",
        "          #print(\"Original: \", predict_prob(actual_word, mask_token_logits), actual_word)\n",
        "          #print(\"Slur: \", predict_prob(word, mask_token_logits), word)\n",
        "\n",
        "          count+=1\n",
        "          if(predict_prob(word, mask_token_logits)<=predict_prob(df['Actual Word'][i], mask_token_logits)):\n",
        "            n_count+=1\n",
        "\n",
        "          dict_, primary_candidates= candidate_set(word, mask_token_logits)\n",
        "          dict_= remove_from_dict(dict_, vid_id, primary_candidates)\n",
        "          dict_=sort_di(dict_)\n",
        "\n",
        "          candidate_list=top_cand(dict_, top_k_words)\n",
        "          #print(candidate_list)\n",
        "\n",
        "          scores.append(find_top(dict_))\n",
        "\n",
        "          ground_truth.append(actual_word)\n",
        "          video_ids_list.append(vid_id)\n",
        "          if(actual_word in candidate_list):\n",
        "            print(\"Present in top candidate list!\")\n",
        "            present.append('Present')\n",
        "            cand_count+=1\n",
        "          else:\n",
        "            present.append('Absent')\n",
        "\n",
        "          #print('\\n\\n\\n')\n",
        "      except Exception as e:\n",
        "       print(e)\n",
        "\n",
        "    print(\"Total: \", count)\n",
        "    print(\"Suggestions that had a greater probability than the slur word: \", n_count)\n",
        "    print(\"Ratio: \", n_count/count)\n",
        "    print(\"Candidate Test Success Count: \", cand_count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LMArszviOzO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f78ce950-a295-4f87-a6ef-c1037c28385e"
      },
      "source": [
        "#For instances\n",
        "scores=[]\n",
        "ground_truth=[]\n",
        "present=[]\n",
        "video_ids_list=[]\n",
        "#This parameters signifies the value of p. topk=1 signifies P-1, topk=3 signifies P-3 and so forth\n",
        "topk=1\n",
        "print_results(df, \"YouTube\", topk)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Present in top candidate list!\n",
            "Present in top candidate list!\n",
            "Present in top candidate list!\n",
            "Present in top candidate list!\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "Present in top candidate list!\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "Present in top candidate list!\n",
            "Present in top candidate list!\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "Present in top candidate list!\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "index 0 is out of bounds for dimension 0 with size 0\n",
            "Total:  323\n",
            "Suggestions that had a greater probability than the slur word:  32\n",
            "Ratio:  0.09907120743034056\n",
            "Candidate Test Success Count:  8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HKyc7ohQihdr"
      },
      "source": [
        "#saving the results into a CSV file\n",
        "list_entries=[]\n",
        "for i in range(len(scores)):\n",
        "  list_entries.append((video_ids_list[i], ground_truth[i], scores[i][0], scores[i][1], present[i]))\n",
        "\n",
        "df_to_save = pd.DataFrame(list_entries, columns =['Video Id', 'Ground Truth','Words','Scores', 'Present'])\n",
        "df_to_save.to_csv(\"CSV/Results/P1_Amazon_Normalized_AddPluralForms.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_to_save"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Xc4mfuE4FzwO",
        "outputId": "c25e939d-ef13-4dd1-b172-3d0b0d2adef4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Video Id   Ground Truth      Words                Scores  Present\n",
              "0   0Z9rIS4xNZ0           maya      [ass]  [2.9170236587524414]   Absent\n",
              "1   Fw1_1_yKZ78         copter    [local]  [0.4974931478500366]   Absent\n",
              "2   pmQTB3ScUo8           crab      [car]   [6.567646026611328]   Absent\n",
              "3   jrZwy-ybyfA           crab      [car]   [7.778393268585205]   Absent\n",
              "4   irlvY_fyNtQ          craft    [craft]  [3.3310887813568115]  Present\n",
              "5   WnijkuQ83TA          craft      [cup]   [2.554880142211914]   Absent\n",
              "6   u5F6eTSvQjw          craft    [craft]  [2.0116078853607178]  Present\n",
              "7   WTdi0S4t-XM          craft      [can]   [1.422349214553833]   Absent\n",
              "8   SnzwfLHGjSg          craft     [trap]   [2.293825149536133]   Absent\n",
              "9   eiIBr_MueIo          craft      [can]   [5.627756118774414]   Absent\n",
              "10  t1ioQ865tBs          craft    [craft]  [4.0767741203308105]  Present\n",
              "11  4YkVBdvP4mw          craft    [craft]  [3.4166064262390137]  Present\n",
              "12  T1ysw09Gq0Q          craft     [trap]   [2.293825149536133]   Absent\n",
              "13  ZphmA0s6M0U         grumpy    [crazy]    [4.13920783996582]   Absent\n",
              "14  gCGKA-HBlrc          adult     [love]   [4.495726585388184]   Absent\n",
              "15  RgSiPh5_jWg        race is     [race]   [3.123422145843506]   Absent\n",
              "16  Nun19Nk5wM4        version   [virgin]     [9.1305513381958]   Absent\n",
              "17  YTSzNyAgL4Q          craft    [craft]  [1.2006224393844604]  Present\n",
              "18  IF79TH-Kar0         dinner    [demon]  [1.0613412857055664]   Absent\n",
              "19  9c8hmzuh-fE        heroine     [hero]  [11.660619735717773]   Absent\n",
              "20  Z85gwsYlPqg        heroine     [hero]  [11.774833679199219]   Absent\n",
              "21  zEGTw8UBWfE        heroine     [hero]  [12.272027969360352]   Absent\n",
              "22  68U6UXbWWLo        version  [version]   [4.794650077819824]  Present\n",
              "23  GlZZSE58c-A        version  [version]   [4.794650077819824]  Present\n",
              "24  4GpYODZCXxk           crab      [car]  [1.0741442441940308]   Absent\n",
              "25  Ux3nTq9miPY          craft     [trap]   [0.933828592300415]   Absent\n",
              "26  S3wU--EIYy8         sleigh    [place]   [4.019305229187012]   Absent\n",
              "27  vIJpOyYIuMk         sleigh    [place]  [2.7264580726623535]   Absent\n",
              "28  TD3RFeUyy7Y         sleigh    [place]   [4.346165180206299]   Absent\n",
              "29  io1wzv13Qw4         sleigh     [same]   [4.419560432434082]   Absent\n",
              "30  8U48-yS94AY  cockadoodledo     [cock]   [4.884635925292969]   Absent\n",
              "31  vu3TJD3660A          craft      [car]  [3.2990071773529053]   Absent\n",
              "32  MFygagQTSCs           Maya       [as]  [2.6012656688690186]   Absent\n",
              "33  1AiEArIlU7E          digly    [chick]   [3.510253429412842]   Absent\n",
              "34  qeLCsSflL7k      round cut    [trunk]   [4.037380695343018]   Absent\n",
              "35  pZzwL7zumfg       Quandong     [Tong]   [6.894805431365967]   Absent\n",
              "36  KYCQVAqDQlY     Dracalaura    [drugs]  [2.0570359230041504]   Absent\n",
              "37  rZ5VQAmBMSU        heroine     [hero]  [11.628707885742188]   Absent\n",
              "38  N10ysIxsS-I          crack      [can]   [3.387289524078369]   Absent\n",
              "39  16IndzFp8HQ          dough      [pop]  [3.4785749912261963]   Absent\n",
              "40  6L55x89fPuI      molecules   [skills]   [5.396559715270996]   Absent\n",
              "41  AcJ583FI88k              s       [as]   [3.796877861022949]   Absent\n",
              "42  7Wfx5Qv1uho           crab     [crab]  [12.027280807495117]  Present\n",
              "43  B6Vaz9OUwfk        version   [virgin]  [12.028017044067383]   Absent"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2d139c57-f570-4966-864e-88824c1031cc\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Video Id</th>\n",
              "      <th>Ground Truth</th>\n",
              "      <th>Words</th>\n",
              "      <th>Scores</th>\n",
              "      <th>Present</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0Z9rIS4xNZ0</td>\n",
              "      <td>maya</td>\n",
              "      <td>[ass]</td>\n",
              "      <td>[2.9170236587524414]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Fw1_1_yKZ78</td>\n",
              "      <td>copter</td>\n",
              "      <td>[local]</td>\n",
              "      <td>[0.4974931478500366]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>pmQTB3ScUo8</td>\n",
              "      <td>crab</td>\n",
              "      <td>[car]</td>\n",
              "      <td>[6.567646026611328]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>jrZwy-ybyfA</td>\n",
              "      <td>crab</td>\n",
              "      <td>[car]</td>\n",
              "      <td>[7.778393268585205]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>irlvY_fyNtQ</td>\n",
              "      <td>craft</td>\n",
              "      <td>[craft]</td>\n",
              "      <td>[3.3310887813568115]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>WnijkuQ83TA</td>\n",
              "      <td>craft</td>\n",
              "      <td>[cup]</td>\n",
              "      <td>[2.554880142211914]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>u5F6eTSvQjw</td>\n",
              "      <td>craft</td>\n",
              "      <td>[craft]</td>\n",
              "      <td>[2.0116078853607178]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>WTdi0S4t-XM</td>\n",
              "      <td>craft</td>\n",
              "      <td>[can]</td>\n",
              "      <td>[1.422349214553833]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>SnzwfLHGjSg</td>\n",
              "      <td>craft</td>\n",
              "      <td>[trap]</td>\n",
              "      <td>[2.293825149536133]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>eiIBr_MueIo</td>\n",
              "      <td>craft</td>\n",
              "      <td>[can]</td>\n",
              "      <td>[5.627756118774414]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>t1ioQ865tBs</td>\n",
              "      <td>craft</td>\n",
              "      <td>[craft]</td>\n",
              "      <td>[4.0767741203308105]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>4YkVBdvP4mw</td>\n",
              "      <td>craft</td>\n",
              "      <td>[craft]</td>\n",
              "      <td>[3.4166064262390137]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>T1ysw09Gq0Q</td>\n",
              "      <td>craft</td>\n",
              "      <td>[trap]</td>\n",
              "      <td>[2.293825149536133]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>ZphmA0s6M0U</td>\n",
              "      <td>grumpy</td>\n",
              "      <td>[crazy]</td>\n",
              "      <td>[4.13920783996582]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>gCGKA-HBlrc</td>\n",
              "      <td>adult</td>\n",
              "      <td>[love]</td>\n",
              "      <td>[4.495726585388184]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>RgSiPh5_jWg</td>\n",
              "      <td>race is</td>\n",
              "      <td>[race]</td>\n",
              "      <td>[3.123422145843506]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Nun19Nk5wM4</td>\n",
              "      <td>version</td>\n",
              "      <td>[virgin]</td>\n",
              "      <td>[9.1305513381958]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>YTSzNyAgL4Q</td>\n",
              "      <td>craft</td>\n",
              "      <td>[craft]</td>\n",
              "      <td>[1.2006224393844604]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>IF79TH-Kar0</td>\n",
              "      <td>dinner</td>\n",
              "      <td>[demon]</td>\n",
              "      <td>[1.0613412857055664]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>9c8hmzuh-fE</td>\n",
              "      <td>heroine</td>\n",
              "      <td>[hero]</td>\n",
              "      <td>[11.660619735717773]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>Z85gwsYlPqg</td>\n",
              "      <td>heroine</td>\n",
              "      <td>[hero]</td>\n",
              "      <td>[11.774833679199219]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>zEGTw8UBWfE</td>\n",
              "      <td>heroine</td>\n",
              "      <td>[hero]</td>\n",
              "      <td>[12.272027969360352]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>68U6UXbWWLo</td>\n",
              "      <td>version</td>\n",
              "      <td>[version]</td>\n",
              "      <td>[4.794650077819824]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>GlZZSE58c-A</td>\n",
              "      <td>version</td>\n",
              "      <td>[version]</td>\n",
              "      <td>[4.794650077819824]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>4GpYODZCXxk</td>\n",
              "      <td>crab</td>\n",
              "      <td>[car]</td>\n",
              "      <td>[1.0741442441940308]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>Ux3nTq9miPY</td>\n",
              "      <td>craft</td>\n",
              "      <td>[trap]</td>\n",
              "      <td>[0.933828592300415]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>S3wU--EIYy8</td>\n",
              "      <td>sleigh</td>\n",
              "      <td>[place]</td>\n",
              "      <td>[4.019305229187012]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>vIJpOyYIuMk</td>\n",
              "      <td>sleigh</td>\n",
              "      <td>[place]</td>\n",
              "      <td>[2.7264580726623535]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>TD3RFeUyy7Y</td>\n",
              "      <td>sleigh</td>\n",
              "      <td>[place]</td>\n",
              "      <td>[4.346165180206299]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>io1wzv13Qw4</td>\n",
              "      <td>sleigh</td>\n",
              "      <td>[same]</td>\n",
              "      <td>[4.419560432434082]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>8U48-yS94AY</td>\n",
              "      <td>cockadoodledo</td>\n",
              "      <td>[cock]</td>\n",
              "      <td>[4.884635925292969]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>vu3TJD3660A</td>\n",
              "      <td>craft</td>\n",
              "      <td>[car]</td>\n",
              "      <td>[3.2990071773529053]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>MFygagQTSCs</td>\n",
              "      <td>Maya</td>\n",
              "      <td>[as]</td>\n",
              "      <td>[2.6012656688690186]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>1AiEArIlU7E</td>\n",
              "      <td>digly</td>\n",
              "      <td>[chick]</td>\n",
              "      <td>[3.510253429412842]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>qeLCsSflL7k</td>\n",
              "      <td>round cut</td>\n",
              "      <td>[trunk]</td>\n",
              "      <td>[4.037380695343018]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>pZzwL7zumfg</td>\n",
              "      <td>Quandong</td>\n",
              "      <td>[Tong]</td>\n",
              "      <td>[6.894805431365967]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>KYCQVAqDQlY</td>\n",
              "      <td>Dracalaura</td>\n",
              "      <td>[drugs]</td>\n",
              "      <td>[2.0570359230041504]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>rZ5VQAmBMSU</td>\n",
              "      <td>heroine</td>\n",
              "      <td>[hero]</td>\n",
              "      <td>[11.628707885742188]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>N10ysIxsS-I</td>\n",
              "      <td>crack</td>\n",
              "      <td>[can]</td>\n",
              "      <td>[3.387289524078369]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>16IndzFp8HQ</td>\n",
              "      <td>dough</td>\n",
              "      <td>[pop]</td>\n",
              "      <td>[3.4785749912261963]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>6L55x89fPuI</td>\n",
              "      <td>molecules</td>\n",
              "      <td>[skills]</td>\n",
              "      <td>[5.396559715270996]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>AcJ583FI88k</td>\n",
              "      <td>s</td>\n",
              "      <td>[as]</td>\n",
              "      <td>[3.796877861022949]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>7Wfx5Qv1uho</td>\n",
              "      <td>crab</td>\n",
              "      <td>[crab]</td>\n",
              "      <td>[12.027280807495117]</td>\n",
              "      <td>Present</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>B6Vaz9OUwfk</td>\n",
              "      <td>version</td>\n",
              "      <td>[virgin]</td>\n",
              "      <td>[12.028017044067383]</td>\n",
              "      <td>Absent</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d139c57-f570-4966-864e-88824c1031cc')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2d139c57-f570-4966-864e-88824c1031cc button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2d139c57-f570-4966-864e-88824c1031cc');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_to_save[df_to_save['Present'] == \"Present\"]) / len(df_to_save)"
      ],
      "metadata": {
        "id": "JszWK_YxHKgC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11d2ad47-ba09-4f74-e48f-022eeb2ae241"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18181818181818182"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DHQFTXpve8Ha"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}