NLP_Pharmacat.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_Pharmacat.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP9kHa81GpokncZgRaTBVbW"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "828f697d02dc4839a9a1b500c2853c16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8182bcad60f94aeeb67f2d874452713a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4dc35be9c29a4119824f33db5437a018",
              "IPY_MODEL_6db6a912b71c4d87afea4f6cf25da256"
            ]
          }
        },
        "8182bcad60f94aeeb67f2d874452713a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4dc35be9c29a4119824f33db5437a018": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ec42493937da474d9aabb8d1de225401",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 405234788,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 405234788,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b50927555da54b63a099c8915900ab12"
          }
        },
        "6db6a912b71c4d87afea4f6cf25da256": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_01e4f3abced94b9da1ffdd976f4d830b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 405M/405M [00:52&lt;00:00, 7.74MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90921d27b7934d1dbfd6be0f19893fb6"
          }
        },
        "ec42493937da474d9aabb8d1de225401": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b50927555da54b63a099c8915900ab12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01e4f3abced94b9da1ffdd976f4d830b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90921d27b7934d1dbfd6be0f19893fb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l8wt_-NwbfEj",
        "outputId": "0847dec9-5ca9-46fd-d7ea-d3d305222bdc"
      },
      "source": [
        "pip install -U sentence-transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentence-transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/9d/abacb6f7bb63df39285c55bb51b6403a7fd93ac2aea48b01f6215175446c/sentence-transformers-1.1.1.tar.gz (81kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 3.5MB/s \n",
            "\u001b[?25hCollecting transformers<5.0.0,>=3.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b0/9e/5b80becd952d5f7250eaf8fc64b957077b12ccfe73e9c03d37146ab29712/transformers-4.6.0-py3-none-any.whl (2.3MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 8.2MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (4.41.1)\n",
            "Requirement already satisfied, skipping upgrade: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.8.1+cu101)\n",
            "Requirement already satisfied, skipping upgrade: torchvision in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (0.9.1+cu101)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.19.5)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (0.22.2.post1)\n",
            "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.4.1)\n",
            "Requirement already satisfied, skipping upgrade: nltk in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 33.1MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 32.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (4.0.1)\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (20.9)\n",
            "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 34.6MB/s \n",
            "\u001b[?25hCollecting huggingface-hub==0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (2019.12.20)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0->sentence-transformers) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision->sentence-transformers) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sentence-transformers) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.7/dist-packages (from nltk->sentence-transformers) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<5.0.0,>=3.1.0->sentence-transformers) (3.4.1)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers<5.0.0,>=3.1.0->sentence-transformers) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=3.1.0->sentence-transformers) (8.0.0)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (2020.12.5)\n",
            "Building wheels for collected packages: sentence-transformers\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-1.1.1-cp37-none-any.whl size=123338 sha256=5c799efa2504a17e727086d9fd80c5680de3f4620e5f0f2492047a5d01532ad1\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/89/29/45e45adc162b50f97f71801e8b07947c9cfe2b3ae7dbf37896\n",
            "Successfully built sentence-transformers\n",
            "Installing collected packages: tokenizers, sacremoses, huggingface-hub, transformers, sentencepiece, sentence-transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 sentence-transformers-1.1.1 sentencepiece-0.1.95 tokenizers-0.10.2 transformers-4.6.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171,
          "referenced_widgets": [
            "828f697d02dc4839a9a1b500c2853c16",
            "8182bcad60f94aeeb67f2d874452713a",
            "4dc35be9c29a4119824f33db5437a018",
            "6db6a912b71c4d87afea4f6cf25da256",
            "ec42493937da474d9aabb8d1de225401",
            "b50927555da54b63a099c8915900ab12",
            "01e4f3abced94b9da1ffdd976f4d830b",
            "90921d27b7934d1dbfd6be0f19893fb6"
          ]
        },
        "id": "20q-1Kdibnt1",
        "outputId": "40e5da56-a897-43ed-84a0-1b1dfc60ad18"
      },
      "source": [
        "import scipy\n",
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('bert-base-nli-mean-tokens')\n",
        "sentences = ['This framework generates embeddings for each input sentence',\n",
        "    'Sentences are passed as a list of string.', \n",
        "    'The quick brown fox jumps over the lazy dog.']\n",
        "sentence_embeddings = model.encode(sentences)\n",
        "\n",
        "print(\"Sentence embeddings:\")\n",
        "print(sentence_embeddings.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "828f697d02dc4839a9a1b500c2853c16",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=405234788.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at /root/.cache/torch/sentence_transformers/sbert.net_models_bert-base-nli-mean-tokens/0_BERT were not used when initializing BertModel: ['classifier.weight', 'classifier.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Sentence embeddings:\n",
            "(3, 768)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Lh1gk1VclIm"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wp1rkB_RdHk5",
        "outputId": "71d50ec5-97a8-4547-e218-93af5ce634e4"
      },
      "source": [
        "data=pd.read_csv('/content/drugsComTrain_raw.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (1,2,3,5) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  interactivity=interactivity, compiler=compiler, result=result)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "cumAYBOzeG_x",
        "outputId": "60cdffc5-bbdd-4b5a-92df-a9638114799c"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uniqueID</th>\n",
              "      <th>drugName</th>\n",
              "      <th>condition</th>\n",
              "      <th>review</th>\n",
              "      <th>rating</th>\n",
              "      <th>date</th>\n",
              "      <th>usefulCount</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>206461.0</td>\n",
              "      <td>Valsartan</td>\n",
              "      <td>Left Ventricular Dysfunction</td>\n",
              "      <td>\"It has no side effect, I take it in combinati...</td>\n",
              "      <td>9.0</td>\n",
              "      <td>20-May-12</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>95260.0</td>\n",
              "      <td>Guanfacine</td>\n",
              "      <td>ADHD</td>\n",
              "      <td>\"My son is halfway through his fourth week of ...</td>\n",
              "      <td>8.0</td>\n",
              "      <td>27-Apr-10</td>\n",
              "      <td>192.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>92703.0</td>\n",
              "      <td>Lybrel</td>\n",
              "      <td>Birth Control</td>\n",
              "      <td>\"I used to take another oral contraceptive, wh...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>14-Dec-09</td>\n",
              "      <td>17.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>138000.0</td>\n",
              "      <td>Ortho Evra</td>\n",
              "      <td>Birth Control</td>\n",
              "      <td>\"This is my first time using any form of birth...</td>\n",
              "      <td>8.0</td>\n",
              "      <td>03-Nov-15</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>35696.0</td>\n",
              "      <td>Buprenorphine / naloxone</td>\n",
              "      <td>Opiate Dependence</td>\n",
              "      <td>\"Suboxone has completely turned my life around...</td>\n",
              "      <td>9.0</td>\n",
              "      <td>27-Nov-16</td>\n",
              "      <td>37.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   uniqueID                  drugName  ...       date usefulCount\n",
              "0  206461.0                 Valsartan  ...  20-May-12        27.0\n",
              "1   95260.0                Guanfacine  ...  27-Apr-10       192.0\n",
              "2   92703.0                    Lybrel  ...  14-Dec-09        17.0\n",
              "3  138000.0                Ortho Evra  ...  03-Nov-15        10.0\n",
              "4   35696.0  Buprenorphine / naloxone  ...  27-Nov-16        37.0\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sc7LtkYzeRgb",
        "outputId": "06e99b44-d667-4376-bbfb-3ecd4d43a6bd"
      },
      "source": [
        "data.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 161296 entries, 0 to 161295\n",
            "Data columns (total 7 columns):\n",
            " #   Column       Non-Null Count  Dtype  \n",
            "---  ------       --------------  -----  \n",
            " 0   uniqueID     27434 non-null  float64\n",
            " 1   drugName     27434 non-null  object \n",
            " 2   condition    27434 non-null  object \n",
            " 3   review       27434 non-null  object \n",
            " 4   rating       27434 non-null  float64\n",
            " 5   date         27434 non-null  object \n",
            " 6   usefulCount  27434 non-null  float64\n",
            "dtypes: float64(3), object(4)\n",
            "memory usage: 8.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mtVh4S1PePc-",
        "outputId": "7c8c9213-f602-4fe3-c37f-da5d5f02aa0a"
      },
      "source": [
        "data.dropna(inplace=True,axis=0)\n",
        "data.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 27434 entries, 0 to 27433\n",
            "Data columns (total 7 columns):\n",
            " #   Column       Non-Null Count  Dtype  \n",
            "---  ------       --------------  -----  \n",
            " 0   uniqueID     27434 non-null  float64\n",
            " 1   drugName     27434 non-null  object \n",
            " 2   condition    27434 non-null  object \n",
            " 3   review       27434 non-null  object \n",
            " 4   rating       27434 non-null  float64\n",
            " 5   date         27434 non-null  object \n",
            " 6   usefulCount  27434 non-null  float64\n",
            "dtypes: float64(3), object(4)\n",
            "memory usage: 1.7+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AE_OEEn2ek6X"
      },
      "source": [
        "complaints=list(data['review'])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UfuFDkn9puLP",
        "outputId": "d552bbb0-662b-476c-da2f-886b61acecd6"
      },
      "source": [
        "#GPU\n",
        "import time\n",
        "s=time.time()\n",
        "complaint_embeddings = model.encode(complaints)\n",
        "time.time()-s"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "97.05251502990723"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MyyBVYXfevLp",
        "outputId": "2f940a0c-10bf-48bb-8004-259233fc97a2"
      },
      "source": [
        "# import time\n",
        "# s=time.time()\n",
        "# complaint_embeddings = model.encode(complaints[:100])\n",
        "# time.time()-s"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60.92678928375244"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HxSiImfBe_-Z",
        "outputId": "660e921a-8aff-4230-903d-1343814990a6"
      },
      "source": [
        "complaint_embeddings.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(27434, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6x9-6U_fU_6"
      },
      "source": [
        "problem='Chest feels heavy and difficulty in breathing, I keep coughing. Sweating alot'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLxAFSUKkjDv"
      },
      "source": [
        "problem_embedding=model.encode(problem)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A9m8lhnJrYOQ",
        "outputId": "50c9ef37-6a25-45d3-e4eb-cc82971a7267"
      },
      "source": [
        "complaint_embeddings[:2,:].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2bH0PGXUkpnV",
        "outputId": "4a93e50e-f673-44f4-af73-3d84935f6d27"
      },
      "source": [
        "np.round(1-scipy.spatial.distance.cdist([problem_embedding], complaint_embeddings[:4,:], \"cosine\"),2)\n",
        "#round((1-scipy.spatial.distance.cdist([problem_embedding], [complaint_embeddings[0]], \"cosine\")[0][0]),2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.13, 0.34, 0.28, 0.29]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAk4tH_FuY9a",
        "outputId": "b032b24f-5415-42ce-d5f1-3d843a82feb4"
      },
      "source": [
        "similarities[:5]\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.13, 0.35, 0.28, 0.28, 0.3 ])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z6o-Rqm8uc8b",
        "outputId": "f10f6d11-8cb5-473a-96ac-f1dea4f6c506"
      },
      "source": [
        "similarities.flatten()[:5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.13, 0.35, 0.28, 0.28, 0.3 ])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dixRMbxtr0oy"
      },
      "source": [
        "similarities=np.round(1-scipy.spatial.distance.cdist([problem_embedding], complaint_embeddings, \"cosine\"),2)\n",
        "similarities=similarities.flatten()\n",
        "similarities_desc=np.sort(similarities)[::-1]\n",
        "#np.argsort(similarities)[::-1]\n",
        "indexes=np.array([i for i in range(complaint_embeddings.shape[0])])\n",
        "best_matches=indexes[np.argsort(similarities)[::-1]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YtSx5tUJntHz",
        "outputId": "a5333692-dfbe-4189-b455-029eb99ad3b1"
      },
      "source": [
        "topn=3\n",
        "print(problem)\n",
        "print()\n",
        "for i in range(topn):\n",
        "  print(data['review'][best_matches[i]],data['condition'][best_matches[i]],similarities[best_matches[i]])\n",
        "  print()\n",
        "  print()\n",
        "\n",
        "  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Chest feels heavy and difficulty in breathing, I keep coughing. Sweating alot\n",
            "\n",
            "\"Dizziness, tiredness, runny nose, sore throat, cough, diarrhea, generally feel bad.\" High Cholesterol 0.8\n",
            "\n",
            "\n",
            "\"I have to take more and more. It wears off fast. I also get constipated and itch like crazy from it!\" Restless Legs Syndrome 0.79\n",
            "\n",
            "\n",
            "\"Was given this for pain, had bad shaking, very anxious and irratable. Also dizzy and nauseated.\" Pain 0.78\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zgS2FYxQxofT",
        "outputId": "96454fb7-84d2-42d6-b251-99edfbdc9300"
      },
      "source": [
        "from sys import getsizeof\n",
        "round(getsizeof(complaint_embeddings) / 1024 / 1024,2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "80.37"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 124
        }
      ]
    }
  ]
}