vllm_prefix_cache.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "L4",
      "authorship_tag": "ABX9TyOUksyJZzi5UcI3HJUe6CK+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ff630df9b9fb4370881ecc48e2ac5f42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e2997e5639843ceb6a4d4671e863022",
              "IPY_MODEL_e8593e12ea124dafbd923fb25b155a3f",
              "IPY_MODEL_de7d98dc12454dfea6ec443fd2d84236"
            ],
            "layout": "IPY_MODEL_3a5c0845ae404a80a4698d7826d412e1"
          }
        },
        "3e2997e5639843ceb6a4d4671e863022": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5b13a70ddee4c4caf452e91a14db696",
            "placeholder": "​",
            "style": "IPY_MODEL_863e57db888d41b28040b4750109fce8",
            "value": ""
          }
        },
        "e8593e12ea124dafbd923fb25b155a3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e531ef36cd8847289ad505e8ae456220",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5148524bc4ab462fa3bc713b74ea043d",
            "value": 2
          }
        },
        "de7d98dc12454dfea6ec443fd2d84236": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b65351224bdd4a20b47537b0fff50955",
            "placeholder": "​",
            "style": "IPY_MODEL_a0d86b45eda94f29a8cd67aa5a9b1876",
            "value": "Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:02&lt;00:00,  1.04s/it]\n"
          }
        },
        "3a5c0845ae404a80a4698d7826d412e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5b13a70ddee4c4caf452e91a14db696": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "863e57db888d41b28040b4750109fce8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e531ef36cd8847289ad505e8ae456220": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5148524bc4ab462fa3bc713b74ea043d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b65351224bdd4a20b47537b0fff50955": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0d86b45eda94f29a8cd67aa5a9b1876": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35f29b1d9bcb4997a0eb65fc4c63b0eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_082f1850b85d47688b541057298b444e",
              "IPY_MODEL_6409f6cc5ff843aa80050cf293cb9b58",
              "IPY_MODEL_2f242b4e3679484fad47f9b801f71554"
            ],
            "layout": "IPY_MODEL_16b82af954f6420e9eb384110ab6c92f"
          }
        },
        "082f1850b85d47688b541057298b444e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e565d54fb18f40cda8b9d8969e16f7c4",
            "placeholder": "​",
            "style": "IPY_MODEL_fdc39ca35b7b44ffbde1af537911e48d",
            "value": "Fetching 11 files: 100%"
          }
        },
        "6409f6cc5ff843aa80050cf293cb9b58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a99a966b53ce471fa9135ee5be14cf07",
            "max": 11,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3bde3f21dfb44066902f80b07a290888",
            "value": 11
          }
        },
        "2f242b4e3679484fad47f9b801f71554": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_976432af5ed4413e8c4053e291050100",
            "placeholder": "​",
            "style": "IPY_MODEL_950248e848344b40b91f73a138c786fc",
            "value": " 11/11 [00:00&lt;00:00, 856.38it/s]"
          }
        },
        "16b82af954f6420e9eb384110ab6c92f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e565d54fb18f40cda8b9d8969e16f7c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdc39ca35b7b44ffbde1af537911e48d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a99a966b53ce471fa9135ee5be14cf07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bde3f21dfb44066902f80b07a290888": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "976432af5ed4413e8c4053e291050100": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "950248e848344b40b91f73a138c786fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "920a2c46db884dd692c10efa03b83d3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_09f2995392f9437f82049ecdfc1914df",
              "IPY_MODEL_d3b86dd2bd6447cdb49dd2f21098c640",
              "IPY_MODEL_fc220c8ee20f40c6bb382a4222e404da"
            ],
            "layout": "IPY_MODEL_e052fd99f7b14d378cb3d1240f9e849e"
          }
        },
        "09f2995392f9437f82049ecdfc1914df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c0c83ac6e2874f109f844c6a2bbca388",
            "placeholder": "​",
            "style": "IPY_MODEL_0c467ec58fb5411bb3a0f313e875ce3e",
            "value": ""
          }
        },
        "d3b86dd2bd6447cdb49dd2f21098c640": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1347799066e4359b784332380688c8a",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a739ff10a7b94c15b857b46c0c5c2412",
            "value": 2
          }
        },
        "fc220c8ee20f40c6bb382a4222e404da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4698889edb2f43b2b2fd9c55d8c9f2ec",
            "placeholder": "​",
            "style": "IPY_MODEL_d11576ef42a849baa44fa73fc656f5b4",
            "value": "Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:02&lt;00:00,  1.04s/it]\n"
          }
        },
        "e052fd99f7b14d378cb3d1240f9e849e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0c83ac6e2874f109f844c6a2bbca388": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c467ec58fb5411bb3a0f313e875ce3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c1347799066e4359b784332380688c8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a739ff10a7b94c15b857b46c0c5c2412": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4698889edb2f43b2b2fd9c55d8c9f2ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d11576ef42a849baa44fa73fc656f5b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jlonge4/gen_ai_utils/blob/main/vllm_prefix_cache.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bHGqs9_857hu"
      },
      "outputs": [],
      "source": [
        "!pip install vllm flash-attention triton peft"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import SamplingParams, LLM\n",
        "\n",
        "texts = [\"God this guy is such a douchebag\",\n",
        "  \"I love bunnies, don't you?\",\n",
        "  \"Hey you dickhead, stop being so stupid\",\n",
        "  \"You're a complete idiot, how did you even get this job?\",\n",
        "  \"I hope you have a wonderful day!\",\n",
        "  \"This product is absolute garbage, don't waste your money\",\n",
        "  \"Your opinion is worthless, just like you\",\n",
        "  \"Thank you for your help, I really appreciate it\",\n",
        "  \"Why don't you go jump off a cliff?\",\n",
        "  \"I respectfully disagree with your point of view\",\n",
        "  \"You're nothing but a useless piece of trash\",\n",
        "  \"Let's work together to find a solution\",\n",
        "  \"I wish you would just disappear forever\"]\n",
        "sampling_params = SamplingParams(temperature=0.01, top_p=0.95, max_tokens=5)"
      ],
      "metadata": {
        "id": "3g_W-Owc6R4l"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def formatting_func(example):\n",
        "    input = f\"\"\"\n",
        "    <|user|>\n",
        "    Examine the text and determine  whether the text is toxic or not. Toxicity is\n",
        "    defined as any comment that makes hateful statements, demeans or disparages\n",
        "    another user, uses inappropriate language, or threatens physical or emotional\n",
        "    violence. Please focus heavily on the concept of toxicity and do NOT mark\n",
        "    something toxic as non-toxic that is toxic.\n",
        "\n",
        "    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\n",
        "    contain any text or characters aside from that word. \"toxic\" means that the text\n",
        "    meets the definition of toxic. \"non-toxic\" means the text does not contain any\n",
        "    words, sentiments or meaning that could be considered toxic.\n",
        "\n",
        "    You are examining written text content. Here is the text:\n",
        "        [BEGIN DATA]\n",
        "        ************\n",
        "        [Text]: {example}\n",
        "        ************\n",
        "        [END DATA]\n",
        "\n",
        "    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\n",
        "    <|end|>\n",
        "    <|assistant|>\n",
        "    \"\"\"\n",
        "    return input"
      ],
      "metadata": {
        "id": "w34co86W6r6i"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompts = []\n",
        "for text in texts:\n",
        "  input = formatting_func(text)\n",
        "  prompts.append(input)"
      ],
      "metadata": {
        "id": "o9pFZfCGQoRr"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm.lora.request import LoRARequest\n",
        "\n",
        "llm = LLM(model=\"microsoft/Phi-3-mini-4k-instruct\", enable_lora=True, gpu_memory_utilization=0.6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451,
          "referenced_widgets": [
            "ff630df9b9fb4370881ecc48e2ac5f42",
            "3e2997e5639843ceb6a4d4671e863022",
            "e8593e12ea124dafbd923fb25b155a3f",
            "de7d98dc12454dfea6ec443fd2d84236",
            "3a5c0845ae404a80a4698d7826d412e1",
            "b5b13a70ddee4c4caf452e91a14db696",
            "863e57db888d41b28040b4750109fce8",
            "e531ef36cd8847289ad505e8ae456220",
            "5148524bc4ab462fa3bc713b74ea043d",
            "b65351224bdd4a20b47537b0fff50955",
            "a0d86b45eda94f29a8cd67aa5a9b1876"
          ]
        },
        "id": "3POidPCO6uf-",
        "outputId": "11f21345-3237-41c6-b866-d4ecaa545bfd"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-19 12:46:55 llm_engine.py:223] Initializing an LLM engine (v0.6.1.post2) with config: model='microsoft/Phi-3-mini-4k-instruct', speculative_config=None, tokenizer='microsoft/Phi-3-mini-4k-instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=microsoft/Phi-3-mini-4k-instruct, use_v2_block_manager=False, num_scheduler_steps=1, enable_prefix_caching=False, use_async_output_proc=True)\n",
            "INFO 09-19 12:46:55 selector.py:240] Cannot use FlashAttention-2 backend due to sliding window.\n",
            "INFO 09-19 12:46:55 selector.py:116] Using XFormers backend.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.\n",
            "  @torch.library.impl_abstract(\"xformers_flash::flash_fwd\")\n",
            "/usr/local/lib/python3.10/dist-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.\n",
            "  @torch.library.impl_abstract(\"xformers_flash::flash_bwd\")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-19 12:46:57 model_runner.py:997] Starting to load model microsoft/Phi-3-mini-4k-instruct...\n",
            "INFO 09-19 12:46:57 selector.py:240] Cannot use FlashAttention-2 backend due to sliding window.\n",
            "INFO 09-19 12:46:57 selector.py:116] Using XFormers backend.\n",
            "INFO 09-19 12:46:58 weight_utils.py:242] Using model weights format ['*.safetensors']\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]\n"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ff630df9b9fb4370881ecc48e2ac5f42"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-19 12:47:01 model_runner.py:1008] Loading model weights took 7.1227 GB\n",
            "INFO 09-19 12:47:04 gpu_executor.py:122] # GPU blocks: 966, # CPU blocks: 682\n",
            "INFO 09-19 12:47:06 model_runner.py:1311] Capturing the model for CUDA graphs. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI.\n",
            "INFO 09-19 12:47:06 model_runner.py:1315] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
            "INFO 09-19 12:47:41 model_runner.py:1430] Graph capturing finished in 35 secs.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import snapshot_download\n",
        "\n",
        "toxic_lora_path = snapshot_download(repo_id=\"grounded-ai/phi3-toxicity-judge\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153,
          "referenced_widgets": [
            "35f29b1d9bcb4997a0eb65fc4c63b0eb",
            "082f1850b85d47688b541057298b444e",
            "6409f6cc5ff843aa80050cf293cb9b58",
            "2f242b4e3679484fad47f9b801f71554",
            "16b82af954f6420e9eb384110ab6c92f",
            "e565d54fb18f40cda8b9d8969e16f7c4",
            "fdc39ca35b7b44ffbde1af537911e48d",
            "a99a966b53ce471fa9135ee5be14cf07",
            "3bde3f21dfb44066902f80b07a290888",
            "976432af5ed4413e8c4053e291050100",
            "950248e848344b40b91f73a138c786fc"
          ]
        },
        "id": "VHJYYQhw7YI0",
        "outputId": "c835a83a-f06d-4f85-e528-1901afbdb4c1"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Fetching 11 files:   0%|          | 0/11 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "35f29b1d9bcb4997a0eb65fc4c63b0eb"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "start_time = time.time()\n",
        "outputs = llm.generate(\n",
        "    prompts,\n",
        "    sampling_params,\n",
        "    lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path)\n",
        ")\n",
        "print(f\"Time taken without prefix: {time.time() - start_time}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WGh5QeRa7NjS",
        "outputId": "ae7de795-b0b2-4489-81d1-e71807b481ce"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-6-4e96696b2a47>:6: DeprecationWarning: The 'lora_local_path' attribute is deprecated and will be removed in a future version. Please use 'lora_path' instead.\n",
            "  lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path)\n",
            "Processed prompts: 100%|██████████| 13/13 [00:00<00:00, 15.14it/s, est. speed input: 3973.36 toks/s, output: 57.06 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time taken without prefix: 0.9476668834686279\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for output in outputs:\n",
        "    prompt = output.prompt\n",
        "    generated_text = output.outputs[0].text\n",
        "    print(f\"Prompt: {prompt!r}, Generated text: {generated_text!r}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J6hXQAEoSxlZ",
        "outputId": "6fefc25d-5417-4d3a-a539-70ba4dcffa9b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: God this guy is such a douchebag\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I love bunnies, don\\'t you?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Hey you dickhead, stop being so stupid\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: You\\'re a complete idiot, how did you even get this job?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I hope you have a wonderful day!\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: This product is absolute garbage, don\\'t waste your money\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Your opinion is worthless, just like you\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Thank you for your help, I really appreciate it\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Why don\\'t you go jump off a cliff?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I respectfully disagree with your point of view\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: You\\'re nothing but a useless piece of trash\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Let\\'s work together to find a solution\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I wish you would just disappear forever\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm.lora.request import LoRARequest\n",
        "llm_prefix_caching = LLM(model=\"microsoft/Phi-3-mini-4k-instruct\", enable_lora=True, gpu_memory_utilization=0.6, enable_prefix_caching=True, disable_sliding_window=True)\n",
        "outputs = llm_prefix_caching.generate(prompts[0],\n",
        "    sampling_params,\n",
        "    lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277,
          "referenced_widgets": [
            "920a2c46db884dd692c10efa03b83d3e",
            "09f2995392f9437f82049ecdfc1914df",
            "d3b86dd2bd6447cdb49dd2f21098c640",
            "fc220c8ee20f40c6bb382a4222e404da",
            "e052fd99f7b14d378cb3d1240f9e849e",
            "c0c83ac6e2874f109f844c6a2bbca388",
            "0c467ec58fb5411bb3a0f313e875ce3e",
            "c1347799066e4359b784332380688c8a",
            "a739ff10a7b94c15b857b46c0c5c2412",
            "4698889edb2f43b2b2fd9c55d8c9f2ec",
            "d11576ef42a849baa44fa73fc656f5b4"
          ]
        },
        "id": "vkb-VnMLQxn2",
        "outputId": "92b39b91-b7f9-4d3f-a7de-5c5c73082509"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-19 12:52:46 llm_engine.py:223] Initializing an LLM engine (v0.6.1.post2) with config: model='microsoft/Phi-3-mini-4k-instruct', speculative_config=None, tokenizer='microsoft/Phi-3-mini-4k-instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=2047, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=microsoft/Phi-3-mini-4k-instruct, use_v2_block_manager=False, num_scheduler_steps=1, enable_prefix_caching=True, use_async_output_proc=True)\n",
            "INFO 09-19 12:52:47 model_runner.py:997] Starting to load model microsoft/Phi-3-mini-4k-instruct...\n",
            "INFO 09-19 12:52:48 weight_utils.py:242] Using model weights format ['*.safetensors']\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]\n"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "920a2c46db884dd692c10efa03b83d3e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-19 12:52:51 model_runner.py:1008] Loading model weights took 7.1227 GB\n",
            "INFO 09-19 12:52:54 gpu_executor.py:122] # GPU blocks: 970, # CPU blocks: 682\n",
            "INFO 09-19 12:52:57 model_runner.py:1311] Capturing the model for CUDA graphs. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI.\n",
            "INFO 09-19 12:52:57 model_runner.py:1315] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
            "INFO 09-19 12:53:30 model_runner.py:1430] Graph capturing finished in 34 secs.\n",
            "INFO 09-19 12:53:30 block_manager_v1.py:263] Automatic prefix caching is enabled.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-5-1605770fa229>:5: DeprecationWarning: The 'lora_local_path' attribute is deprecated and will be removed in a future version. Please use 'lora_path' instead.\n",
            "  lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path))\n",
            "Processed prompts: 100%|██████████| 1/1 [00:00<00:00,  3.23it/s, est. speed input: 852.07 toks/s, output: 9.72 toks/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "start_time = time.time()\n",
        "outputs = llm_prefix_caching.generate(\n",
        "    prompts,\n",
        "    sampling_params,\n",
        "    lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path)\n",
        ")\n",
        "print(f\"Time taken with prefix cache: {time.time() - start_time}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4QbNSs55Rk9n",
        "outputId": "3044f013-f668-43e3-8541-9a73918510d9"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-6-90f5340793a5>:6: DeprecationWarning: The 'lora_local_path' attribute is deprecated and will be removed in a future version. Please use 'lora_path' instead.\n",
            "  lora_request=LoRARequest(\"toxicity_adapter\", 1, toxic_lora_path)\n",
            "Processed prompts: 100%|██████████| 13/13 [00:00<00:00, 24.43it/s, est. speed input: 6413.61 toks/s, output: 92.10 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time taken with prefix cache: 0.54833984375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for output in outputs:\n",
        "    prompt = output.prompt\n",
        "    generated_text = output.outputs[0].text\n",
        "    print(f\"Prompt: {prompt!r}, Generated text: {generated_text!r}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DbGMLygPSJsN",
        "outputId": "ffccc7a7-95b7-4cb0-8ab3-a0c2a4a2b612"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: God this guy is such a douchebag\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I love bunnies, don\\'t you?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Hey you dickhead, stop being so stupid\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: You\\'re a complete idiot, how did you even get this job?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I hope you have a wonderful day!\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: This product is absolute garbage, don\\'t waste your money\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Your opinion is worthless, just like you\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Thank you for your help, I really appreciate it\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Why don\\'t you go jump off a cliff?\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I respectfully disagree with your point of view\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: You\\'re nothing but a useless piece of trash\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: Let\\'s work together to find a solution\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' non-toxic'\n",
            "Prompt: '\\n    <|user|>\\n    Examine the text and determine  whether the text is toxic or not. Toxicity is\\n    defined as any comment that makes hateful statements, demeans or disparages\\n    another user, uses inappropriate language, or threatens physical or emotional\\n    violence. Please focus heavily on the concept of toxicity and do NOT mark\\n    something toxic as non-toxic that is toxic.\\n\\n    Your response must be single word, either \"toxic\" or \"non-toxic\", and should not\\n    contain any text or characters aside from that word. \"toxic\" means that the text\\n    meets the definition of toxic. \"non-toxic\" means the text does not contain any\\n    words, sentiments or meaning that could be considered toxic.\\n\\n    You are examining written text content. Here is the text:\\n        [BEGIN DATA]\\n        ************\\n        [Text]: I wish you would just disappear forever\\n        ************\\n        [END DATA]\\n\\n    You must only reply with a single word: either \"toxic\", or \"non-toxic\".\\n    <|end|>\\n    <|assistant|>\\n    ', Generated text: ' toxic'\n"
          ]
        }
      ]
    }
  ]
}