engineering_monkeypox_mvp.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/slp22/data-engineering-project/blob/main/engineering_monkeypox_mvp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wH8m3rmEAo3U"
      },
      "source": [
        "#### Data Engineering | MVP\n",
        "\n",
        "# Monkeypox Tweets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4nm_mHUuAeql"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "w06wbr_xATv4"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import logging\n",
        "import sqlite3\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os, shutil, itertools\n",
        "import pandas as pd\n",
        "import pathlib as Path\n",
        "import pickle\n",
        "import PIL\n",
        "import random\n",
        "import seaborn as sns\n",
        "import sklearn as sk\n",
        "import warnings\n",
        "import zipfile\n",
        "\n",
        "from sqlite3 import connect\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Google Drive imports"
      ],
      "metadata": {
        "id": "g8kqta7It40w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# mount drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# authorize access \n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "# read in from Google Sheets\n",
        "\n",
        "import gspread\n",
        "from google.auth import default\n",
        "creds, _ = default()\n",
        "\n",
        "gc = gspread.authorize(creds)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KZF_QxCLs0Vf",
        "outputId": "dd50c116-c184-4f3a-96b5-cde0c461e9e1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# copy kaggle key from Google Drive\n",
        "shutil.copyfile('/content/drive/MyDrive/kaggle.json', '/content/kaggle.json')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "xmUmjfBGGwMS",
        "outputId": "01b6e9a5-91a8-40f7-bb30-f5fdba573577"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/kaggle.json'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! chmod 600 /content/kaggle.json"
      ],
      "metadata": {
        "id": "LdnEQ2dyRTOC"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3uJyuCs9jkJ7",
        "outputId": "ba83982c-31e4-4366-d422-c44ff0104a7d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.config', 'kaggle.json', 'drive', 'sample_data']"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "#confirm kaggle key is in dir\n",
        "os.listdir()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "ure1COVBG3sw"
      },
      "outputs": [],
      "source": [
        "with open('kaggle.json') as json_file:\n",
        "    kaggle = json.load(json_file)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "As3njbbDmjAY"
      },
      "outputs": [],
      "source": [
        "# assign to directory \n",
        "os.environ['KAGGLE_CONFIG_DIR'] = \"/content\""
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### py/spark\n",
        "\n"
      ],
      "metadata": {
        "id": "B386_PwAwSxC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # https://towardsdatascience.com/pyspark-on-google-colab-101-d31830b238be\n",
        "# # https://www.analyticsvidhya.com/blog/2020/11/a-must-read-guide-on-how-to-work-with-pyspark-on-google-colab-for-data-scientists/\n",
        "\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null"
      ],
      "metadata": {
        "id": "d6zyxbIeQRP7"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -q https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop3.2.tgz"
      ],
      "metadata": {
        "id": "PnrT-MdkO4Om"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!tar -xf spark-3.0.0-bin-hadoop3.2.tgz"
      ],
      "metadata": {
        "id": "Fh2JdMTItP4v"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q findspark"
      ],
      "metadata": {
        "id": "jM9TqdQ0up5A"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.0.0-bin-hadoop3.2\""
      ],
      "metadata": {
        "id": "kiB4yJevOiGk"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "findspark.init()"
      ],
      "metadata": {
        "id": "DNzHHgiROoIL"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pyspark\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import split, explode, col, lower\n",
        "from pyspark.sql.types import StructType,StructField, StringType, IntegerType\n"
      ],
      "metadata": {
        "id": "Ocg1qNbOuwSo"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehG7rDKVA1x3"
      },
      "source": [
        "## 1 | Research Design\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lPrQPhqxA4C4"
      },
      "source": [
        "* **Research Question:** Can we build a dashboard to monitor top trending topics on Twitter about monkeypox?\n",
        "* **Data source:** [Kaggle Tweets on Monkeypox ](https://www.kaggle.com/datasets/aneeshtickoo/tweets-on-monkeypox) (n=6,410)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j9xSJjZAA79Y"
      },
      "source": [
        "## 2 | Data Ingestion"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### [Monkeypox Tweets](https://www.kaggle.com/datasets/aneeshtickoo/tweets-on-monkeypox)"
      ],
      "metadata": {
        "id": "jXIfCh8OaYtc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "jD2q1R_8movA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c57adf5a-92b1-4794-87c0-ee3fc16d17d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading tweets-on-monkeypox.zip to /content\n",
            "\r  0% 0.00/1.12M [00:00<?, ?B/s]\n",
            "\r100% 1.12M/1.12M [00:00<00:00, 140MB/s]\n"
          ]
        }
      ],
      "source": [
        "# download dataset from kaggle\n",
        "! kaggle datasets download -d aneeshtickoo/tweets-on-monkeypox"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "8bszkBF7ndyj"
      },
      "outputs": [],
      "source": [
        "# unzip kaggle file\n",
        "zip_ref = zipfile.ZipFile('tweets-on-monkeypox.zip', 'r') \n",
        "zip_ref.extractall('/content')\n",
        "zip_ref.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "678fce99"
      },
      "source": [
        "## 3 | Exploratory Data Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### First glance"
      ],
      "metadata": {
        "id": "Oscqdm5WZ3Ap"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "rpMzt1LuEr_S"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('/content/monkeypox.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "UX-TC5yOHcbY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "7526b759-21e2-4d54-a843-efdab9ccfeb1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                    id      conversation_id  \\\n",
              "0  1555815462201872385  1555815462201872385   \n",
              "1  1555815458602831872  1555815458602831872   \n",
              "\n",
              "                                created_at        date      time  timezone  \\\n",
              "0  2022-08-06 12:48:06 India Standard Time  2022-08-06  12:48:06       530   \n",
              "1  2022-08-06 12:48:05 India Standard Time  2022-08-06  12:48:05       530   \n",
              "\n",
              "              user_id      username                           name place  ...  \\\n",
              "0  820113517613154304  thetenth2022                       TheTenth   NaN  ...   \n",
              "1           196518052     ashemedai  Jeroen Ruigrok van der Werven   NaN  ...   \n",
              "\n",
              "  geo source user_rt_id user_rt retweet_id  reply_to  retweet_date  translate  \\\n",
              "0 NaN    NaN        NaN     NaN        NaN        []           NaN        NaN   \n",
              "1 NaN    NaN        NaN     NaN        NaN        []           NaN        NaN   \n",
              "\n",
              "  trans_src trans_dest  \n",
              "0       NaN        NaN  \n",
              "1       NaN        NaN  \n",
              "\n",
              "[2 rows x 36 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8ed5959f-5b08-4d75-b60f-c4db6583ecc4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conversation_id</th>\n",
              "      <th>created_at</th>\n",
              "      <th>date</th>\n",
              "      <th>time</th>\n",
              "      <th>timezone</th>\n",
              "      <th>user_id</th>\n",
              "      <th>username</th>\n",
              "      <th>name</th>\n",
              "      <th>place</th>\n",
              "      <th>...</th>\n",
              "      <th>geo</th>\n",
              "      <th>source</th>\n",
              "      <th>user_rt_id</th>\n",
              "      <th>user_rt</th>\n",
              "      <th>retweet_id</th>\n",
              "      <th>reply_to</th>\n",
              "      <th>retweet_date</th>\n",
              "      <th>translate</th>\n",
              "      <th>trans_src</th>\n",
              "      <th>trans_dest</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1555815462201872385</td>\n",
              "      <td>1555815462201872385</td>\n",
              "      <td>2022-08-06 12:48:06 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:06</td>\n",
              "      <td>530</td>\n",
              "      <td>820113517613154304</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>TheTenth</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1555815458602831872</td>\n",
              "      <td>1555815458602831872</td>\n",
              "      <td>2022-08-06 12:48:05 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:05</td>\n",
              "      <td>530</td>\n",
              "      <td>196518052</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>Jeroen Ruigrok van der Werven</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2 rows × 36 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8ed5959f-5b08-4d75-b60f-c4db6583ecc4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8ed5959f-5b08-4d75-b60f-c4db6583ecc4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8ed5959f-5b08-4d75-b60f-c4db6583ecc4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "df.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "YjhK5W3wE9gu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5acf494b-2ff2-4c12-bf93-224e48258323"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 6859 entries, 0 to 6858\n",
            "Data columns (total 36 columns):\n",
            " #   Column           Non-Null Count  Dtype  \n",
            "---  ------           --------------  -----  \n",
            " 0   id               6859 non-null   int64  \n",
            " 1   conversation_id  6859 non-null   int64  \n",
            " 2   created_at       6859 non-null   object \n",
            " 3   date             6859 non-null   object \n",
            " 4   time             6859 non-null   object \n",
            " 5   timezone         6859 non-null   int64  \n",
            " 6   user_id          6859 non-null   int64  \n",
            " 7   username         6859 non-null   object \n",
            " 8   name             6859 non-null   object \n",
            " 9   place            2 non-null      object \n",
            " 10  tweet            6859 non-null   object \n",
            " 11  language         6859 non-null   object \n",
            " 12  mentions         6859 non-null   object \n",
            " 13  urls             6859 non-null   object \n",
            " 14  photos           6859 non-null   object \n",
            " 15  replies_count    6859 non-null   int64  \n",
            " 16  retweets_count   6859 non-null   int64  \n",
            " 17  likes_count      6859 non-null   int64  \n",
            " 18  hashtags         6859 non-null   object \n",
            " 19  cashtags         6859 non-null   object \n",
            " 20  link             6859 non-null   object \n",
            " 21  retweet          6859 non-null   bool   \n",
            " 22  quote_url        587 non-null    object \n",
            " 23  video            6859 non-null   int64  \n",
            " 24  thumbnail        658 non-null    object \n",
            " 25  near             0 non-null      float64\n",
            " 26  geo              0 non-null      float64\n",
            " 27  source           0 non-null      float64\n",
            " 28  user_rt_id       0 non-null      float64\n",
            " 29  user_rt          0 non-null      float64\n",
            " 30  retweet_id       0 non-null      float64\n",
            " 31  reply_to         6859 non-null   object \n",
            " 32  retweet_date     0 non-null      float64\n",
            " 33  translate        0 non-null      float64\n",
            " 34  trans_src        0 non-null      float64\n",
            " 35  trans_dest       0 non-null      float64\n",
            "dtypes: bool(1), float64(10), int64(8), object(17)\n",
            "memory usage: 1.8+ MB\n"
          ]
        }
      ],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "qKXgvoqwHkPF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "8e92c438-19a3-4e17-e7d8-a07a6a3f37e9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 id  conversation_id  timezone       user_id  replies_count  \\\n",
              "count  6.859000e+03     6.859000e+03    6859.0  6.859000e+03    6859.000000   \n",
              "mean   1.555764e+18     1.555192e+18     530.0  7.224884e+17       0.399038   \n",
              "std    2.541498e+13     1.168003e+16       0.0  6.620987e+17       1.789475   \n",
              "min    1.555725e+18     8.510359e+17     530.0  3.989300e+04       0.000000   \n",
              "25%    1.555742e+18     1.555720e+18     530.0  4.906582e+08       0.000000   \n",
              "50%    1.555760e+18     1.555746e+18     530.0  8.977771e+17       0.000000   \n",
              "75%    1.555784e+18     1.555774e+18     530.0  1.395422e+18       0.000000   \n",
              "max    1.555815e+18     1.555815e+18     530.0  1.555798e+18      68.000000   \n",
              "\n",
              "       retweets_count  likes_count        video  near  geo  source  \\\n",
              "count     6859.000000  6859.000000  6859.000000   0.0  0.0     0.0   \n",
              "mean         0.691938     3.053944     0.095932   NaN  NaN     NaN   \n",
              "std         11.879167    40.567536     0.294520   NaN  NaN     NaN   \n",
              "min          0.000000     0.000000     0.000000   NaN  NaN     NaN   \n",
              "25%          0.000000     0.000000     0.000000   NaN  NaN     NaN   \n",
              "50%          0.000000     0.000000     0.000000   NaN  NaN     NaN   \n",
              "75%          0.000000     1.000000     0.000000   NaN  NaN     NaN   \n",
              "max        740.000000  2180.000000     1.000000   NaN  NaN     NaN   \n",
              "\n",
              "       user_rt_id  user_rt  retweet_id  retweet_date  translate  trans_src  \\\n",
              "count         0.0      0.0         0.0           0.0        0.0        0.0   \n",
              "mean          NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "std           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "min           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "25%           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "50%           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "75%           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "max           NaN      NaN         NaN           NaN        NaN        NaN   \n",
              "\n",
              "       trans_dest  \n",
              "count         0.0  \n",
              "mean          NaN  \n",
              "std           NaN  \n",
              "min           NaN  \n",
              "25%           NaN  \n",
              "50%           NaN  \n",
              "75%           NaN  \n",
              "max           NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-89b4da09-3768-43a9-b0bf-9441aeebb862\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conversation_id</th>\n",
              "      <th>timezone</th>\n",
              "      <th>user_id</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>retweets_count</th>\n",
              "      <th>likes_count</th>\n",
              "      <th>video</th>\n",
              "      <th>near</th>\n",
              "      <th>geo</th>\n",
              "      <th>source</th>\n",
              "      <th>user_rt_id</th>\n",
              "      <th>user_rt</th>\n",
              "      <th>retweet_id</th>\n",
              "      <th>retweet_date</th>\n",
              "      <th>translate</th>\n",
              "      <th>trans_src</th>\n",
              "      <th>trans_dest</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>6.859000e+03</td>\n",
              "      <td>6.859000e+03</td>\n",
              "      <td>6859.0</td>\n",
              "      <td>6.859000e+03</td>\n",
              "      <td>6859.000000</td>\n",
              "      <td>6859.000000</td>\n",
              "      <td>6859.000000</td>\n",
              "      <td>6859.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.555764e+18</td>\n",
              "      <td>1.555192e+18</td>\n",
              "      <td>530.0</td>\n",
              "      <td>7.224884e+17</td>\n",
              "      <td>0.399038</td>\n",
              "      <td>0.691938</td>\n",
              "      <td>3.053944</td>\n",
              "      <td>0.095932</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.541498e+13</td>\n",
              "      <td>1.168003e+16</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6.620987e+17</td>\n",
              "      <td>1.789475</td>\n",
              "      <td>11.879167</td>\n",
              "      <td>40.567536</td>\n",
              "      <td>0.294520</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.555725e+18</td>\n",
              "      <td>8.510359e+17</td>\n",
              "      <td>530.0</td>\n",
              "      <td>3.989300e+04</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>1.555742e+18</td>\n",
              "      <td>1.555720e+18</td>\n",
              "      <td>530.0</td>\n",
              "      <td>4.906582e+08</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.555760e+18</td>\n",
              "      <td>1.555746e+18</td>\n",
              "      <td>530.0</td>\n",
              "      <td>8.977771e+17</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.555784e+18</td>\n",
              "      <td>1.555774e+18</td>\n",
              "      <td>530.0</td>\n",
              "      <td>1.395422e+18</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.555815e+18</td>\n",
              "      <td>1.555815e+18</td>\n",
              "      <td>530.0</td>\n",
              "      <td>1.555798e+18</td>\n",
              "      <td>68.000000</td>\n",
              "      <td>740.000000</td>\n",
              "      <td>2180.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-89b4da09-3768-43a9-b0bf-9441aeebb862')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-89b4da09-3768-43a9-b0bf-9441aeebb862 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-89b4da09-3768-43a9-b0bf-9441aeebb862');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "df.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "E9KmD81nEwgq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc548060-3ebe-4231-8e46-ef058949651c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['id',\n",
              " 'conversation_id',\n",
              " 'created_at',\n",
              " 'date',\n",
              " 'time',\n",
              " 'timezone',\n",
              " 'user_id',\n",
              " 'username',\n",
              " 'name',\n",
              " 'place',\n",
              " 'tweet',\n",
              " 'language',\n",
              " 'mentions',\n",
              " 'urls',\n",
              " 'photos',\n",
              " 'replies_count',\n",
              " 'retweets_count',\n",
              " 'likes_count',\n",
              " 'hashtags',\n",
              " 'cashtags',\n",
              " 'link',\n",
              " 'retweet',\n",
              " 'quote_url',\n",
              " 'video',\n",
              " 'thumbnail',\n",
              " 'near',\n",
              " 'geo',\n",
              " 'source',\n",
              " 'user_rt_id',\n",
              " 'user_rt',\n",
              " 'retweet_id',\n",
              " 'reply_to',\n",
              " 'retweet_date',\n",
              " 'translate',\n",
              " 'trans_src',\n",
              " 'trans_dest']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "cols_list = list(df.columns)\n",
        "cols_list"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "7cFIjuFSW1dZ",
        "outputId": "ebf0be75-32e7-44e0-bf69-5008480a2069"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       id      conversation_id  \\\n",
              "0     1555815462201872385  1555815462201872385   \n",
              "1     1555815458602831872  1555815458602831872   \n",
              "2     1555815444359135235  1555815444359135235   \n",
              "3     1555815437409157120  1555815437409157120   \n",
              "4     1555815424666869761  1555595696639774722   \n",
              "...                   ...                  ...   \n",
              "6854  1555725304659935232  1555718508813570048   \n",
              "6855  1555725295432245248  1555725295432245248   \n",
              "6856  1555725294685900801  1555655517850128384   \n",
              "6857  1555725290055385089  1555725290055385089   \n",
              "6858  1555725288201494529  1555725288201494529   \n",
              "\n",
              "                                   created_at        date      time  timezone  \\\n",
              "0     2022-08-06 12:48:06 India Standard Time  2022-08-06  12:48:06       530   \n",
              "1     2022-08-06 12:48:05 India Standard Time  2022-08-06  12:48:05       530   \n",
              "2     2022-08-06 12:48:02 India Standard Time  2022-08-06  12:48:02       530   \n",
              "3     2022-08-06 12:48:00 India Standard Time  2022-08-06  12:48:00       530   \n",
              "4     2022-08-06 12:47:57 India Standard Time  2022-08-06  12:47:57       530   \n",
              "...                                       ...         ...       ...       ...   \n",
              "6854  2022-08-06 06:49:51 India Standard Time  2022-08-06  06:49:51       530   \n",
              "6855  2022-08-06 06:49:49 India Standard Time  2022-08-06  06:49:49       530   \n",
              "6856  2022-08-06 06:49:48 India Standard Time  2022-08-06  06:49:48       530   \n",
              "6857  2022-08-06 06:49:47 India Standard Time  2022-08-06  06:49:47       530   \n",
              "6858  2022-08-06 06:49:47 India Standard Time  2022-08-06  06:49:47       530   \n",
              "\n",
              "                  user_id         username                           name  \\\n",
              "0      820113517613154304     thetenth2022                       TheTenth   \n",
              "1               196518052        ashemedai  Jeroen Ruigrok van der Werven   \n",
              "2               600467609  democracymotion            Democracy In Motion   \n",
              "3     1431886953998258177      thegoogle93                     The Google   \n",
              "4     1506745578159591428   bufflosouljah1                  Bufflosouljah   \n",
              "...                   ...              ...                            ...   \n",
              "6854  1498464506384732162     arkcowgirl62                  ValTheCowgirl   \n",
              "6855             38338051     speehanagram                        Spencer   \n",
              "6856  1548821540577464322   pritzkertoilet         Rev. Strychnine Twitch   \n",
              "6857            298690857     thumbressler                      Mike Reux   \n",
              "6858           4069820847  nurse4allpeople              (((Love4Planet)))   \n",
              "\n",
              "     place  ... geo source user_rt_id user_rt retweet_id  \\\n",
              "0      NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "1      NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "2      NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "3      NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "4      NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "...    ...  ...  ..    ...        ...     ...        ...   \n",
              "6854   NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "6855   NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "6856   NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "6857   NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "6858   NaN  ... NaN    NaN        NaN     NaN        NaN   \n",
              "\n",
              "                                               reply_to  retweet_date  \\\n",
              "0                                                    []           NaN   \n",
              "1                                                    []           NaN   \n",
              "2                                                    []           NaN   \n",
              "3                                                    []           NaN   \n",
              "4     [{'screen_name': 'Natrone86', 'name': 'Weed Al...           NaN   \n",
              "...                                                 ...           ...   \n",
              "6854  [{'screen_name': 'hurtmeknots', 'name': \"Sambe...           NaN   \n",
              "6855                                                 []           NaN   \n",
              "6856  [{'screen_name': 'politvidchannel', 'name': 'P...           NaN   \n",
              "6857                                                 []           NaN   \n",
              "6858                                                 []           NaN   \n",
              "\n",
              "      translate trans_src trans_dest  \n",
              "0           NaN       NaN        NaN  \n",
              "1           NaN       NaN        NaN  \n",
              "2           NaN       NaN        NaN  \n",
              "3           NaN       NaN        NaN  \n",
              "4           NaN       NaN        NaN  \n",
              "...         ...       ...        ...  \n",
              "6854        NaN       NaN        NaN  \n",
              "6855        NaN       NaN        NaN  \n",
              "6856        NaN       NaN        NaN  \n",
              "6857        NaN       NaN        NaN  \n",
              "6858        NaN       NaN        NaN  \n",
              "\n",
              "[6859 rows x 36 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e2f8a45d-d107-494e-a38f-7e81320c66d7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>conversation_id</th>\n",
              "      <th>created_at</th>\n",
              "      <th>date</th>\n",
              "      <th>time</th>\n",
              "      <th>timezone</th>\n",
              "      <th>user_id</th>\n",
              "      <th>username</th>\n",
              "      <th>name</th>\n",
              "      <th>place</th>\n",
              "      <th>...</th>\n",
              "      <th>geo</th>\n",
              "      <th>source</th>\n",
              "      <th>user_rt_id</th>\n",
              "      <th>user_rt</th>\n",
              "      <th>retweet_id</th>\n",
              "      <th>reply_to</th>\n",
              "      <th>retweet_date</th>\n",
              "      <th>translate</th>\n",
              "      <th>trans_src</th>\n",
              "      <th>trans_dest</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1555815462201872385</td>\n",
              "      <td>1555815462201872385</td>\n",
              "      <td>2022-08-06 12:48:06 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:06</td>\n",
              "      <td>530</td>\n",
              "      <td>820113517613154304</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>TheTenth</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1555815458602831872</td>\n",
              "      <td>1555815458602831872</td>\n",
              "      <td>2022-08-06 12:48:05 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:05</td>\n",
              "      <td>530</td>\n",
              "      <td>196518052</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>Jeroen Ruigrok van der Werven</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1555815444359135235</td>\n",
              "      <td>1555815444359135235</td>\n",
              "      <td>2022-08-06 12:48:02 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:02</td>\n",
              "      <td>530</td>\n",
              "      <td>600467609</td>\n",
              "      <td>democracymotion</td>\n",
              "      <td>Democracy In Motion</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1555815437409157120</td>\n",
              "      <td>1555815437409157120</td>\n",
              "      <td>2022-08-06 12:48:00 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:48:00</td>\n",
              "      <td>530</td>\n",
              "      <td>1431886953998258177</td>\n",
              "      <td>thegoogle93</td>\n",
              "      <td>The Google</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1555815424666869761</td>\n",
              "      <td>1555595696639774722</td>\n",
              "      <td>2022-08-06 12:47:57 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>12:47:57</td>\n",
              "      <td>530</td>\n",
              "      <td>1506745578159591428</td>\n",
              "      <td>bufflosouljah1</td>\n",
              "      <td>Bufflosouljah</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'Natrone86', 'name': 'Weed Al...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6854</th>\n",
              "      <td>1555725304659935232</td>\n",
              "      <td>1555718508813570048</td>\n",
              "      <td>2022-08-06 06:49:51 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>06:49:51</td>\n",
              "      <td>530</td>\n",
              "      <td>1498464506384732162</td>\n",
              "      <td>arkcowgirl62</td>\n",
              "      <td>ValTheCowgirl</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'hurtmeknots', 'name': \"Sambe...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6855</th>\n",
              "      <td>1555725295432245248</td>\n",
              "      <td>1555725295432245248</td>\n",
              "      <td>2022-08-06 06:49:49 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>06:49:49</td>\n",
              "      <td>530</td>\n",
              "      <td>38338051</td>\n",
              "      <td>speehanagram</td>\n",
              "      <td>Spencer</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6856</th>\n",
              "      <td>1555725294685900801</td>\n",
              "      <td>1555655517850128384</td>\n",
              "      <td>2022-08-06 06:49:48 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>06:49:48</td>\n",
              "      <td>530</td>\n",
              "      <td>1548821540577464322</td>\n",
              "      <td>pritzkertoilet</td>\n",
              "      <td>Rev. Strychnine Twitch</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[{'screen_name': 'politvidchannel', 'name': 'P...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6857</th>\n",
              "      <td>1555725290055385089</td>\n",
              "      <td>1555725290055385089</td>\n",
              "      <td>2022-08-06 06:49:47 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>06:49:47</td>\n",
              "      <td>530</td>\n",
              "      <td>298690857</td>\n",
              "      <td>thumbressler</td>\n",
              "      <td>Mike Reux</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6858</th>\n",
              "      <td>1555725288201494529</td>\n",
              "      <td>1555725288201494529</td>\n",
              "      <td>2022-08-06 06:49:47 India Standard Time</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>06:49:47</td>\n",
              "      <td>530</td>\n",
              "      <td>4069820847</td>\n",
              "      <td>nurse4allpeople</td>\n",
              "      <td>(((Love4Planet)))</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6859 rows × 36 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e2f8a45d-d107-494e-a38f-7e81320c66d7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e2f8a45d-d107-494e-a38f-7e81320c66d7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e2f8a45d-d107-494e-a38f-7e81320c66d7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y8bzxErCFMH0"
      },
      "source": [
        "* RangeIndex: 6859 entries, 0 to 6858\n",
        "* Data columns (total 36 columns)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### word count"
      ],
      "metadata": {
        "id": "KQGIpl2swXl1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# type(df)\n",
        "# df.head(2)\n",
        "pandas_df = df[['date','tweet']]\n",
        "pandas_df.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "-NxZ4zSXVABr",
        "outputId": "5efee382-5da2-490e-b231-7f38bb7b86a4"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         date                                              tweet\n",
              "0  2022-08-06  So has anyone begun to compile 'here's the sta...\n",
              "1  2022-08-06  Getting some groceries, topic shifted to covid..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-873b6bbd-6a26-4db5-8401-8b5011504ba0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>So has anyone begun to compile 'here's the sta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>Getting some groceries, topic shifted to covid...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-873b6bbd-6a26-4db5-8401-8b5011504ba0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-873b6bbd-6a26-4db5-8401-8b5011504ba0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-873b6bbd-6a26-4db5-8401-8b5011504ba0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder\\\n",
        "        .master(\"local\")\\\n",
        "        .appName(\"Colab\")\\\n",
        "        .config('spark.ui.port', '4050')\\\n",
        "        .getOrCreate()"
      ],
      "metadata": {
        "id": "4UfNrKdtuwU9"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_schema = StructType([StructField(\"date\", StringType(), True),\n",
        "                        StructField(\"tweet\", StringType(), True)])\n"
      ],
      "metadata": {
        "id": "rgWhL5t9V6OV"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# tweet_df = spark.read.csv('/content/tweets.csv', inferSchema=True, header='true')\n",
        "\n",
        "spark_df = spark.createDataFrame(pandas_df, schema=df_schema)\n",
        "spark_df.printSchema()\n",
        "spark_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z6O2lw8KRkGV",
        "outputId": "60cea236-d791-4524-84ec-e61896634e26"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- date: string (nullable = true)\n",
            " |-- tweet: string (nullable = true)\n",
            "\n",
            "+----------+--------------------+\n",
            "|      date|               tweet|\n",
            "+----------+--------------------+\n",
            "|2022-08-06|So has anyone beg...|\n",
            "|2022-08-06|Getting some groc...|\n",
            "|2022-08-06|\"Illinois Childre...|\n",
            "|2022-08-06|Illinois daycare ...|\n",
            "|2022-08-06|@Natrone86 @Junot...|\n",
            "|2022-08-06|\"CDC Recommends L...|\n",
            "|2022-08-06|1,058,637 America...|\n",
            "|2022-08-06|MF didn’t know sh...|\n",
            "|2022-08-06|Three people of A...|\n",
            "|2022-08-06|MonkeyPox Case: ರ...|\n",
            "|2022-08-06|#Monkeypox Fallza...|\n",
            "|2022-08-06|RT-PCR detection ...|\n",
            "|2022-08-06|@BigBadDenis When...|\n",
            "|2022-08-06|News Outlook TOP ...|\n",
            "|2022-08-06|CDC Issues Eye-Op...|\n",
            "|2022-08-06|United States dec...|\n",
            "|2022-08-06|@FoxNews They are...|\n",
            "|2022-08-06|@PetiteNicoco No....|\n",
            "|2022-08-06|Idc if it’s Monke...|\n",
            "|2022-08-06|In the technical ...|\n",
            "+----------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark_df = spark_df.withColumn('tweet', \n",
        "                               explode(split(lower(col('tweet')), '\\s')))"
      ],
      "metadata": {
        "id": "Xi934J5ppqJg"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(spark_df.groupBy('tweet')\n",
        "  .count()\n",
        "  .orderBy('count', ascending=False)\n",
        "  .show(10))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9n6mq63EpqP-",
        "outputId": "d6dbb419-3908-460d-9941-cf9049352362"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+-----+\n",
            "|    tweet|count|\n",
            "+---------+-----+\n",
            "|         | 4898|\n",
            "|      the| 4351|\n",
            "|monkeypox| 3230|\n",
            "|       to| 2957|\n",
            "|        a| 2400|\n",
            "|      and| 2344|\n",
            "|   monkey| 2151|\n",
            "|       is| 2014|\n",
            "|       of| 1901|\n",
            "|        i| 1721|\n",
            "+---------+-----+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(spark_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rhLht-zO18Qc",
        "outputId": "0e110865-e027-490b-aa22-d4cbbd0e8b6e"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[date: string, tweet: string]"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###❓Count specific words using PySpark?[link text](https://)"
      ],
      "metadata": {
        "id": "kZHBKBPlX4qW"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gOaVANqDPdyW"
      },
      "source": [
        "### Corpus `tweets`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "P1dHMD7eINnP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "9e233651-254f-4a9a-cd0f-325543dfd86c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  language        date      username hashtags  \\\n",
              "0       en  2022-08-06  thetenth2022       []   \n",
              "1       en  2022-08-06     ashemedai       []   \n",
              "\n",
              "                                               tweet  \n",
              "0  So has anyone begun to compile 'here's the sta...  \n",
              "1  Getting some groceries, topic shifted to covid...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-84aa598f-ef98-4584-8fd6-66f364042757\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>language</th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>en</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>[]</td>\n",
              "      <td>So has anyone begun to compile 'here's the sta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>en</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>[]</td>\n",
              "      <td>Getting some groceries, topic shifted to covid...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-84aa598f-ef98-4584-8fd6-66f364042757')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-84aa598f-ef98-4584-8fd6-66f364042757 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-84aa598f-ef98-4584-8fd6-66f364042757');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ],
      "source": [
        "tweets = df[['language','date','username','hashtags','tweet']]\n",
        "tweets.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "FVfBoFtpKRAc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60f3bf43-27cf-4f24-a46a-56ee1f9765fd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 6859 entries, 0 to 6858\n",
            "Data columns (total 5 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   language  6859 non-null   object\n",
            " 1   date      6859 non-null   object\n",
            " 2   username  6859 non-null   object\n",
            " 3   hashtags  6859 non-null   object\n",
            " 4   tweet     6859 non-null   object\n",
            "dtypes: object(5)\n",
            "memory usage: 268.1+ KB\n"
          ]
        }
      ],
      "source": [
        "tweets.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "wHSEPHPRKqVm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a1e27b87-913c-4a06-a406-62b13b7773f9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6028"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ],
      "source": [
        "tweets['username'].nunique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "eObt84ozIrbo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4f4a4aec-bee8-4b64-cb35-deeeeaabd5a7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['en', 'kn', 'de', 'in', 'tl', 'fr', 'pt', 'te', 'tr', 'it', 'qme',\n",
              "       'bn', 'qht', 'pl', 'es', 'el', 'nl', 'cy', 'ta', 'hi', 'sv', 'ja',\n",
              "       'th', 'mr', 'et', 'gu', 'da', 'ro', 'ml', 'zxx', 'und', 'pa', 'ur',\n",
              "       'ko', 'am', 'fi', 'zh', 'lt', 'hu', 'ru', 'ar', 'si'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ],
      "source": [
        "# How many languages? \n",
        "tweets['language'].unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "85bBZU97JxGI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd97a924-ca7a-46c6-8bbe-02a864a5d37f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English entries: 6410\n",
            "Spanish entries: 51\n",
            "Italian entries: 10\n"
          ]
        }
      ],
      "source": [
        "# How many tweets in English?\n",
        "print('English entries:', (tweets[tweets[\"language\"] == 'en'].count())['language'])\n",
        "\n",
        "# How many tweets in other languages?\n",
        "print('Spanish entries:', (tweets[tweets[\"language\"] == 'es'].count())['language'])\n",
        "print('Italian entries:', (tweets[tweets[\"language\"] == 'it'].count())['language'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "J_yic6fZLT1m"
      },
      "outputs": [],
      "source": [
        "# Keep only English languge tweets\n",
        "tweets = tweets[(tweets['language'] == 'en')]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tweets.info()"
      ],
      "metadata": {
        "id": "KszlRGQ_bXeB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd6d2a3d-9636-4c09-e75c-42602b9c6ef7"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 6410 entries, 0 to 6858\n",
            "Data columns (total 5 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   language  6410 non-null   object\n",
            " 1   date      6410 non-null   object\n",
            " 2   username  6410 non-null   object\n",
            " 3   hashtags  6410 non-null   object\n",
            " 4   tweet     6410 non-null   object\n",
            "dtypes: object(5)\n",
            "memory usage: 300.5+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "BsHh6TcUKxY5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "66c551a4-badf-4673-fb8a-7e6111f2cbc7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  language        date      username hashtags  \\\n",
              "0       en  2022-08-06  thetenth2022       []   \n",
              "1       en  2022-08-06     ashemedai       []   \n",
              "\n",
              "                                               tweet  \n",
              "0  So has anyone begun to compile 'here's the sta...  \n",
              "1  Getting some groceries, topic shifted to covid...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-549671d5-d90b-4ed5-9bdf-04ccc320a972\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>language</th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>en</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>[]</td>\n",
              "      <td>So has anyone begun to compile 'here's the sta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>en</td>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>[]</td>\n",
              "      <td>Getting some groceries, topic shifted to covid...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-549671d5-d90b-4ed5-9bdf-04ccc320a972')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-549671d5-d90b-4ed5-9bdf-04ccc320a972 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-549671d5-d90b-4ed5-9bdf-04ccc320a972');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ],
      "source": [
        "tweets.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "qXlvCWuLQX4t"
      },
      "outputs": [],
      "source": [
        "# Drop language column since we have only English tweets\n",
        "tweets = tweets.drop(columns=['language'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tweets.head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "WTh8IT2aIbtr",
        "outputId": "a6a9e6ce-e7d2-468e-a10f-b76d8666ada1"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          date         username hashtags  \\\n",
              "0   2022-08-06     thetenth2022       []   \n",
              "1   2022-08-06        ashemedai       []   \n",
              "2   2022-08-06  democracymotion       []   \n",
              "3   2022-08-06      thegoogle93       []   \n",
              "4   2022-08-06   bufflosouljah1       []   \n",
              "5   2022-08-06  democracymotion       []   \n",
              "6   2022-08-06       salemjakes       []   \n",
              "7   2022-08-06      fineassbrei       []   \n",
              "8   2022-08-06         xenohadi       []   \n",
              "11  2022-08-06   geopoliticsind       []   \n",
              "\n",
              "                                                tweet  \n",
              "0   So has anyone begun to compile 'here's the sta...  \n",
              "1   Getting some groceries, topic shifted to covid...  \n",
              "2   \"Illinois Children's Daycare Worker Tests Posi...  \n",
              "3   Illinois daycare worker tests positive for mon...  \n",
              "4   @Natrone86 @JunotIsrael @elcavaqueen @thechicc...  \n",
              "5   \"CDC Recommends Limiting Sex Partners to Avoid...  \n",
              "6   1,058,637 Americans have died in the U.S. Covi...  \n",
              "7   MF didn’t know shit about monkey pox in June c...  \n",
              "8   Three people of African origin still admitted ...  \n",
              "11  RT-PCR detection kit for human monkeypox virus...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-12b68e2d-645c-465a-a5b9-3cf374420c23\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>[]</td>\n",
              "      <td>So has anyone begun to compile 'here's the sta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>[]</td>\n",
              "      <td>Getting some groceries, topic shifted to covid...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>democracymotion</td>\n",
              "      <td>[]</td>\n",
              "      <td>\"Illinois Children's Daycare Worker Tests Posi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thegoogle93</td>\n",
              "      <td>[]</td>\n",
              "      <td>Illinois daycare worker tests positive for mon...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>bufflosouljah1</td>\n",
              "      <td>[]</td>\n",
              "      <td>@Natrone86 @JunotIsrael @elcavaqueen @thechicc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>democracymotion</td>\n",
              "      <td>[]</td>\n",
              "      <td>\"CDC Recommends Limiting Sex Partners to Avoid...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>salemjakes</td>\n",
              "      <td>[]</td>\n",
              "      <td>1,058,637 Americans have died in the U.S. Covi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>fineassbrei</td>\n",
              "      <td>[]</td>\n",
              "      <td>MF didn’t know shit about monkey pox in June c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>xenohadi</td>\n",
              "      <td>[]</td>\n",
              "      <td>Three people of African origin still admitted ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>geopoliticsind</td>\n",
              "      <td>[]</td>\n",
              "      <td>RT-PCR detection kit for human monkeypox virus...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-12b68e2d-645c-465a-a5b9-3cf374420c23')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-12b68e2d-645c-465a-a5b9-3cf374420c23 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-12b68e2d-645c-465a-a5b9-3cf374420c23');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "Okq1YwhUP5J5"
      },
      "outputs": [],
      "source": [
        "# Save corpus\n",
        "tweets.to_pickle('/content/tweets.pkl')\n",
        "tweets.to_csv(r'/content/tweets.csv', index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 4 | Storage"
      ],
      "metadata": {
        "id": "5WYNxkC8Vhd0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### SQL Database `monkeypox.db`"
      ],
      "metadata": {
        "id": "SZKYDeyjaST2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### helper functions"
      ],
      "metadata": {
        "id": "WosK0haUgjFI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# https://towardsdatascience.com/have-a-sql-interview-coming-up-ace-it-using-google-colab-6d3c0ffb29dc\n",
        "\n",
        "def pd_to_sqlDB(input_df: pd.DataFrame,\n",
        "                table_name: str,\n",
        "                db_name: str = 'default.db') -> None:\n",
        "\n",
        "    # # Setup local logging\n",
        "    # logging.basicConfig(level=logging.INFO,\n",
        "    #                     format='%(asctime)s %(levelname)s: %(message)s',\n",
        "    #                     datefmt='%Y-%m-%d %H:%M:%S')\n",
        "\n",
        "    # Find columns in the dataframe\n",
        "    cols = input_df.columns\n",
        "    cols_string = ','.join(cols)\n",
        "    val_wildcard_string = ','.join(['?'] * len(cols))\n",
        "\n",
        "    # Connect to a DB file if it exists, else create a new file\n",
        "    con = sqlite3.connect(db_name)\n",
        "    cur = con.cursor()\n",
        "    # logging.info(f'SQL DB {db_name} created')\n",
        "\n",
        "    # Create table\n",
        "    sql_string = f\"\"\"CREATE TABLE {table_name} ({cols_string});\"\"\"\n",
        "    cur.execute(sql_string)\n",
        "    # logging.info(f'SQL Table {table_name} created with {len(cols)} columns')\n",
        "\n",
        "    # Upload df\n",
        "    rows_to_upload = input_df.to_dict(orient='split')['data']\n",
        "    sql_string = f\"\"\"INSERT INTO {table_name} ({cols_string}) VALUES ({val_wildcard_string});\"\"\"    \n",
        "    cur.executemany(sql_string, rows_to_upload)\n",
        "    # logging.info(f'{len(rows_to_upload)} rows uploaded to {table_name}')\n",
        "  \n",
        "    # Commit the changes and close the connection\n",
        "    con.commit()\n",
        "    con.close()"
      ],
      "metadata": {
        "id": "nkPEnnFeRdWZ"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  https://towardsdatascience.com/have-a-sql-interview-coming-up-ace-it-using-google-colab-6d3c0ffb29dc\n",
        "\n",
        "def sql_query_to_pd(sql_query_string: str, db_name: str ='mpox.db') -> pd.DataFrame:\n",
        "    \n",
        "    # Connect to the SQL DB\n",
        "    con = sqlite3.connect(db_name)\n",
        "\n",
        "    # Execute the SQL query\n",
        "    cursor = con.execute(sql_query_string)\n",
        "\n",
        "    # Fetch the data and column names\n",
        "    result_data = cursor.fetchall()\n",
        "    cols = [description[0] for description in cursor.description]\n",
        "\n",
        "    # Close the connection\n",
        "    con.close()\n",
        "\n",
        "    # Return as df\n",
        "    return pd.DataFrame(result_data, columns=cols)"
      ],
      "metadata": {
        "id": "NyBhsOFhP8qM"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### sql_to_df"
      ],
      "metadata": {
        "id": "4GerWOi2gpZf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# https://towardsdatascience.com/have-a-sql-interview-coming-up-ace-it-using-google-colab-6d3c0ffb29dc\n",
        "\n",
        "# Read  csv as df\n",
        "input_df = pd.read_csv('/content/tweets.csv')\n",
        "\n",
        "# Upload df to a SQL table\n",
        "pd_to_sqlDB(input_df,\n",
        "            table_name='tweets',\n",
        "            db_name='monkeypox.db')\n",
        "\n",
        "# Write SQL query in a string variable\n",
        "sql_query_string = \"\"\"\n",
        "    SELECT *\n",
        "    FROM tweets\n",
        "\"\"\"\n",
        "# Exectue  SQL query\n",
        "corpus = sql_query_to_pd(sql_query_string, db_name='monkeypox.db')\n",
        "corpus"
      ],
      "metadata": {
        "id": "-VBhUeOYQi1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "6f56e40f-554c-4eee-88dc-facb44ce844e"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            date         username                         hashtags  \\\n",
              "0     2022-08-06     thetenth2022                               []   \n",
              "1     2022-08-06        ashemedai                               []   \n",
              "2     2022-08-06  democracymotion                               []   \n",
              "3     2022-08-06      thegoogle93                               []   \n",
              "4     2022-08-06   bufflosouljah1                               []   \n",
              "...          ...              ...                              ...   \n",
              "6405  2022-08-06     arkcowgirl62                               []   \n",
              "6406  2022-08-06     speehanagram          ['monkeypox', 'besafe']   \n",
              "6407  2022-08-06   pritzkertoilet                               []   \n",
              "6408  2022-08-06     thumbressler                    ['monkeypox']   \n",
              "6409  2022-08-06  nurse4allpeople  ['moscowmitch', 'donothinggqp']   \n",
              "\n",
              "                                                  tweet  \n",
              "0     So has anyone begun to compile 'here's the sta...  \n",
              "1     Getting some groceries, topic shifted to covid...  \n",
              "2     \"Illinois Children's Daycare Worker Tests Posi...  \n",
              "3     Illinois daycare worker tests positive for mon...  \n",
              "4     @Natrone86 @JunotIsrael @elcavaqueen @thechicc...  \n",
              "...                                                 ...  \n",
              "6405  @hurtmeknots With Monkeypox on the rise, maybe...  \n",
              "6406  @FITNESSSF can you start filling up the disinf...  \n",
              "6407     @politvidchannel 71% already have monkey pox .  \n",
              "6408                          RT if you have #monkeypox  \n",
              "6409  @POTUS @VP 🥰 Dems 🇺🇸💙 are having the best jobs...  \n",
              "\n",
              "[6410 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6d04e074-8b50-4d10-94e6-5f08bd675971\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>username</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thetenth2022</td>\n",
              "      <td>[]</td>\n",
              "      <td>So has anyone begun to compile 'here's the sta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>ashemedai</td>\n",
              "      <td>[]</td>\n",
              "      <td>Getting some groceries, topic shifted to covid...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>democracymotion</td>\n",
              "      <td>[]</td>\n",
              "      <td>\"Illinois Children's Daycare Worker Tests Posi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thegoogle93</td>\n",
              "      <td>[]</td>\n",
              "      <td>Illinois daycare worker tests positive for mon...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>bufflosouljah1</td>\n",
              "      <td>[]</td>\n",
              "      <td>@Natrone86 @JunotIsrael @elcavaqueen @thechicc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6405</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>arkcowgirl62</td>\n",
              "      <td>[]</td>\n",
              "      <td>@hurtmeknots With Monkeypox on the rise, maybe...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6406</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>speehanagram</td>\n",
              "      <td>['monkeypox', 'besafe']</td>\n",
              "      <td>@FITNESSSF can you start filling up the disinf...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6407</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>pritzkertoilet</td>\n",
              "      <td>[]</td>\n",
              "      <td>@politvidchannel 71% already have monkey pox .</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6408</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>thumbressler</td>\n",
              "      <td>['monkeypox']</td>\n",
              "      <td>RT if you have #monkeypox</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6409</th>\n",
              "      <td>2022-08-06</td>\n",
              "      <td>nurse4allpeople</td>\n",
              "      <td>['moscowmitch', 'donothinggqp']</td>\n",
              "      <td>@POTUS @VP 🥰 Dems 🇺🇸💙 are having the best jobs...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6410 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6d04e074-8b50-4d10-94e6-5f08bd675971')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6d04e074-8b50-4d10-94e6-5f08bd675971 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6d04e074-8b50-4d10-94e6-5f08bd675971');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* **database = `monkeypox.db`**\n",
        "* **table_1 = `corpus` (6410 rows × 4 columns)**"
      ],
      "metadata": {
        "id": "kliu0uXRZEWN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# copy corpus csv \n",
        "shutil.copyfile('/content/tweets.csv', '/content/drive/MyDrive/tweets.csv')"
      ],
      "metadata": {
        "id": "TH8hOxQwJBgp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "fb226497-bb9d-4549-b2af-eb112235de69"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/drive/MyDrive/tweets.csv'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 5 | Processing"
      ],
      "metadata": {
        "id": "o58AqssjdG6J"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "* natural language processing, topic modeling (LDA)  \n",
        "* symptom trends across time + moving average (% or |n|) "
      ],
      "metadata": {
        "id": "oUBlwVm-daGH"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Case Counts by State"
      ],
      "metadata": {
        "id": "S5iwVAVOOUlQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import sheet with state lattitude and longitude \n",
        "# data source: https://developers.google.com/public-data/docs/canonical/states_csv\n",
        "\n",
        "# open google spreadsheet\n",
        "worksheet = gc.open('USA-State-Coordinates').sheet1\n",
        "\n",
        "# get_all_values gives a list of rows.\n",
        "rows = worksheet.get_all_values()\n",
        "\n",
        "states = pd.DataFrame.from_records(rows)\n",
        "\n",
        "states.columns = states.iloc[0]\n",
        "states.drop([0], inplace=True)\n",
        "states.drop(['state'], axis=1, inplace=True)\n",
        "states.sort_values(by=['name'], inplace=True)\n",
        "states = states.rename(columns={'latitude': 'lat', 'longitude': 'lon', 'name': 'state'})\n",
        "\n",
        "states.tail(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "tVDmYyswr4QE",
        "outputId": "0c828b3c-39a0-4df1-de0e-1162a818920b"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         lat          lon      state\n",
              "51   43.78444   -88.787868  Wisconsin\n",
              "52  43.075968  -107.290284    Wyoming"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-24fb545c-6714-4de4-b9b8-2765d8867103\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>lat</th>\n",
              "      <th>lon</th>\n",
              "      <th>state</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>43.78444</td>\n",
              "      <td>-88.787868</td>\n",
              "      <td>Wisconsin</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>43.075968</td>\n",
              "      <td>-107.290284</td>\n",
              "      <td>Wyoming</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-24fb545c-6714-4de4-b9b8-2765d8867103')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-24fb545c-6714-4de4-b9b8-2765d8867103 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-24fb545c-6714-4de4-b9b8-2765d8867103');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "states.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eK6GEsrz8-NP",
        "outputId": "823a0aab-eecf-419d-bc12-990a396edad7"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 52 entries, 1 to 52\n",
            "Data columns (total 3 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   lat     52 non-null     object\n",
            " 1   lon     52 non-null     object\n",
            " 2   state   52 non-null     object\n",
            "dtypes: object(3)\n",
            "memory usage: 1.6+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# import sheet with US case count by state \n",
        "# data source: https://www.cdc.gov/poxvirus/monkeypox/response/2022/us-map.html\n",
        "\n",
        "worksheet = gc.open('2022-US-mpx-cases-by-state').sheet1\n",
        "rows = worksheet.get_all_values()\n",
        "cases = pd.DataFrame.from_records(rows)\n",
        "\n",
        "cases.columns = cases.iloc[0]\n",
        "cases.drop([0], inplace=True)\n",
        "cases.drop(['AsOf', 'Case Range'], axis=1, inplace=True)\n",
        "cases.sort_values(by=['Location'], inplace=True)\n",
        "\n",
        "cases.tail(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "I3LNNbQNr4Ut",
        "outputId": "3913cf22-dc4d-414c-af68-80402e5a1cf5"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    Location Cases\n",
              "51  Wisconsin    56\n",
              "52    Wyoming     1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6fb26747-5e4a-402d-a52c-b0869733f360\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Location</th>\n",
              "      <th>Cases</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>Wisconsin</td>\n",
              "      <td>56</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>Wyoming</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6fb26747-5e4a-402d-a52c-b0869733f360')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6fb26747-5e4a-402d-a52c-b0869733f360 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6fb26747-5e4a-402d-a52c-b0869733f360');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# US case count by state (long + lat)\n",
        "map_data = pd.concat([states, cases], axis=1)\n",
        "map_data = map_data[['Location', 'Cases', 'lat', 'lon' ]]\n",
        "map_data = map_data.rename(columns={'Location': 'state', 'Cases':'cases'})\n",
        "\n",
        "map_data.tail(2)\n",
        "# map_data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "iBEYh5Gyr4Xj",
        "outputId": "372d2438-0dd5-4947-d595-fb4bbcaf959e"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       state cases        lat          lon\n",
              "51  Wisconsin    56   43.78444   -88.787868\n",
              "52    Wyoming     1  43.075968  -107.290284"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4af10529-5051-41df-8564-30c0d771a95e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>state</th>\n",
              "      <th>cases</th>\n",
              "      <th>lat</th>\n",
              "      <th>lon</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>Wisconsin</td>\n",
              "      <td>56</td>\n",
              "      <td>43.78444</td>\n",
              "      <td>-88.787868</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>Wyoming</td>\n",
              "      <td>1</td>\n",
              "      <td>43.075968</td>\n",
              "      <td>-107.290284</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4af10529-5051-41df-8564-30c0d771a95e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4af10529-5051-41df-8564-30c0d771a95e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4af10529-5051-41df-8564-30c0d771a95e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "map_data = map_data.astype({'cases':'int'})\n",
        "\n",
        "map_data.sort_values(by=['cases'], ascending=False)\n",
        "# map_data.sort_values(by=['latitude'], ascending=False)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "KmWfhCDOr4dB",
        "outputId": "ac06f447-a563-4585-cb36-e73cecf7fb15"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                  state  cases        lat          lon\n",
              "5             California   3291  36.778261  -119.417932\n",
              "33              New York   3124  43.299428   -74.217933\n",
              "10               Florida   1739  27.664827   -81.515754\n",
              "45                 Texas   1472  31.968599   -99.901813\n",
              "11               Georgia   1299  32.157435   -82.907123\n",
              "14              Illinois   1005  40.633125   -89.398528\n",
              "31            New Jersey    479  40.058324   -74.405661\n",
              "39          Pennsylvania    477  41.203322   -77.194525\n",
              "21              Maryland    461  39.045755   -76.641271\n",
              "9   District of Columbia    414  38.905985   -77.033418\n",
              "49            Washington    332  47.751074  -120.740139\n",
              "48              Virginia    302  37.431573   -78.656894\n",
              "34        North Carolina    288  35.759573     -79.0193\n",
              "3                Arizona    281  34.048928  -111.093731\n",
              "22         Massachusetts    280  42.407211   -71.382437\n",
              "6               Colorado    189  39.550051  -105.782067\n",
              "23              Michigan    165  44.314844   -85.602364\n",
              "19             Louisiana    162  31.244823   -92.145024\n",
              "44             Tennessee    156  35.517491   -86.580447\n",
              "36                  Ohio    151  40.417287   -82.907123\n",
              "15               Indiana    147  40.551217   -85.602364\n",
              "38                Oregon    141  43.804133  -120.554201\n",
              "29                Nevada    127   38.80261  -116.419389\n",
              "24             Minnesota    109  46.729553     -94.6859\n",
              "42        South Carolina    100  33.836081   -81.163725\n",
              "40           Puerto Rico     97  18.220833   -66.590149\n",
              "46                  Utah     93   39.32098  -111.093731\n",
              "7            Connecticut     89  41.603221   -73.087749\n",
              "51             Wisconsin     56   43.78444   -88.787868\n",
              "1                Alabama     53  32.318231   -86.902298\n",
              "26              Missouri     42  37.964253   -91.831833\n",
              "41          Rhode Island     42  41.580095   -71.477429\n",
              "25           Mississippi     29  32.354668   -89.398528\n",
              "4               Arkansas     28   35.20105   -91.831833\n",
              "18              Kentucky     27  37.839333   -84.270018\n",
              "28              Nebraska     25  41.492537   -99.901813\n",
              "32            New Mexico     23   34.97273  -105.032363\n",
              "37              Oklahoma     22  35.007752   -97.092877\n",
              "8               Delaware     21  38.910832    -75.52767\n",
              "30         New Hampshire     19  43.193852   -71.572395\n",
              "16                  Iowa     18  41.878003   -93.097702\n",
              "12                Hawaii     15  19.898682  -155.665857\n",
              "13                 Idaho      9  44.068202  -114.742041\n",
              "17                Kansas      5  39.011902   -98.484246\n",
              "50         West Virginia      5  38.597626   -80.454903\n",
              "20                 Maine      5  45.253783   -69.445469\n",
              "35          North Dakota      4  47.551493  -101.002012\n",
              "27               Montana      3  46.879682  -110.362566\n",
              "2                 Alaska      3  63.588753  -154.493062\n",
              "47               Vermont      2  44.558803   -72.577841\n",
              "43          South Dakota      2  43.969515   -99.901813\n",
              "52               Wyoming      1  43.075968  -107.290284"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-71f9613d-f62a-4616-854c-cb3be0dc3b4c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>state</th>\n",
              "      <th>cases</th>\n",
              "      <th>lat</th>\n",
              "      <th>lon</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>California</td>\n",
              "      <td>3291</td>\n",
              "      <td>36.778261</td>\n",
              "      <td>-119.417932</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>New York</td>\n",
              "      <td>3124</td>\n",
              "      <td>43.299428</td>\n",
              "      <td>-74.217933</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Florida</td>\n",
              "      <td>1739</td>\n",
              "      <td>27.664827</td>\n",
              "      <td>-81.515754</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>Texas</td>\n",
              "      <td>1472</td>\n",
              "      <td>31.968599</td>\n",
              "      <td>-99.901813</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Georgia</td>\n",
              "      <td>1299</td>\n",
              "      <td>32.157435</td>\n",
              "      <td>-82.907123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Illinois</td>\n",
              "      <td>1005</td>\n",
              "      <td>40.633125</td>\n",
              "      <td>-89.398528</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>New Jersey</td>\n",
              "      <td>479</td>\n",
              "      <td>40.058324</td>\n",
              "      <td>-74.405661</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>Pennsylvania</td>\n",
              "      <td>477</td>\n",
              "      <td>41.203322</td>\n",
              "      <td>-77.194525</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>Maryland</td>\n",
              "      <td>461</td>\n",
              "      <td>39.045755</td>\n",
              "      <td>-76.641271</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>District of Columbia</td>\n",
              "      <td>414</td>\n",
              "      <td>38.905985</td>\n",
              "      <td>-77.033418</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>Washington</td>\n",
              "      <td>332</td>\n",
              "      <td>47.751074</td>\n",
              "      <td>-120.740139</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>Virginia</td>\n",
              "      <td>302</td>\n",
              "      <td>37.431573</td>\n",
              "      <td>-78.656894</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>North Carolina</td>\n",
              "      <td>288</td>\n",
              "      <td>35.759573</td>\n",
              "      <td>-79.0193</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Arizona</td>\n",
              "      <td>281</td>\n",
              "      <td>34.048928</td>\n",
              "      <td>-111.093731</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>Massachusetts</td>\n",
              "      <td>280</td>\n",
              "      <td>42.407211</td>\n",
              "      <td>-71.382437</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Colorado</td>\n",
              "      <td>189</td>\n",
              "      <td>39.550051</td>\n",
              "      <td>-105.782067</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>Michigan</td>\n",
              "      <td>165</td>\n",
              "      <td>44.314844</td>\n",
              "      <td>-85.602364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Louisiana</td>\n",
              "      <td>162</td>\n",
              "      <td>31.244823</td>\n",
              "      <td>-92.145024</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>Tennessee</td>\n",
              "      <td>156</td>\n",
              "      <td>35.517491</td>\n",
              "      <td>-86.580447</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>Ohio</td>\n",
              "      <td>151</td>\n",
              "      <td>40.417287</td>\n",
              "      <td>-82.907123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Indiana</td>\n",
              "      <td>147</td>\n",
              "      <td>40.551217</td>\n",
              "      <td>-85.602364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>Oregon</td>\n",
              "      <td>141</td>\n",
              "      <td>43.804133</td>\n",
              "      <td>-120.554201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>Nevada</td>\n",
              "      <td>127</td>\n",
              "      <td>38.80261</td>\n",
              "      <td>-116.419389</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>Minnesota</td>\n",
              "      <td>109</td>\n",
              "      <td>46.729553</td>\n",
              "      <td>-94.6859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>South Carolina</td>\n",
              "      <td>100</td>\n",
              "      <td>33.836081</td>\n",
              "      <td>-81.163725</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>Puerto Rico</td>\n",
              "      <td>97</td>\n",
              "      <td>18.220833</td>\n",
              "      <td>-66.590149</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>Utah</td>\n",
              "      <td>93</td>\n",
              "      <td>39.32098</td>\n",
              "      <td>-111.093731</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Connecticut</td>\n",
              "      <td>89</td>\n",
              "      <td>41.603221</td>\n",
              "      <td>-73.087749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>Wisconsin</td>\n",
              "      <td>56</td>\n",
              "      <td>43.78444</td>\n",
              "      <td>-88.787868</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Alabama</td>\n",
              "      <td>53</td>\n",
              "      <td>32.318231</td>\n",
              "      <td>-86.902298</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>Missouri</td>\n",
              "      <td>42</td>\n",
              "      <td>37.964253</td>\n",
              "      <td>-91.831833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>Rhode Island</td>\n",
              "      <td>42</td>\n",
              "      <td>41.580095</td>\n",
              "      <td>-71.477429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>Mississippi</td>\n",
              "      <td>29</td>\n",
              "      <td>32.354668</td>\n",
              "      <td>-89.398528</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Arkansas</td>\n",
              "      <td>28</td>\n",
              "      <td>35.20105</td>\n",
              "      <td>-91.831833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Kentucky</td>\n",
              "      <td>27</td>\n",
              "      <td>37.839333</td>\n",
              "      <td>-84.270018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>Nebraska</td>\n",
              "      <td>25</td>\n",
              "      <td>41.492537</td>\n",
              "      <td>-99.901813</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>New Mexico</td>\n",
              "      <td>23</td>\n",
              "      <td>34.97273</td>\n",
              "      <td>-105.032363</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>Oklahoma</td>\n",
              "      <td>22</td>\n",
              "      <td>35.007752</td>\n",
              "      <td>-97.092877</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Delaware</td>\n",
              "      <td>21</td>\n",
              "      <td>38.910832</td>\n",
              "      <td>-75.52767</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>New Hampshire</td>\n",
              "      <td>19</td>\n",
              "      <td>43.193852</td>\n",
              "      <td>-71.572395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Iowa</td>\n",
              "      <td>18</td>\n",
              "      <td>41.878003</td>\n",
              "      <td>-93.097702</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Hawaii</td>\n",
              "      <td>15</td>\n",
              "      <td>19.898682</td>\n",
              "      <td>-155.665857</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Idaho</td>\n",
              "      <td>9</td>\n",
              "      <td>44.068202</td>\n",
              "      <td>-114.742041</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Kansas</td>\n",
              "      <td>5</td>\n",
              "      <td>39.011902</td>\n",
              "      <td>-98.484246</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>West Virginia</td>\n",
              "      <td>5</td>\n",
              "      <td>38.597626</td>\n",
              "      <td>-80.454903</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>Maine</td>\n",
              "      <td>5</td>\n",
              "      <td>45.253783</td>\n",
              "      <td>-69.445469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>North Dakota</td>\n",
              "      <td>4</td>\n",
              "      <td>47.551493</td>\n",
              "      <td>-101.002012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>Montana</td>\n",
              "      <td>3</td>\n",
              "      <td>46.879682</td>\n",
              "      <td>-110.362566</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Alaska</td>\n",
              "      <td>3</td>\n",
              "      <td>63.588753</td>\n",
              "      <td>-154.493062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>Vermont</td>\n",
              "      <td>2</td>\n",
              "      <td>44.558803</td>\n",
              "      <td>-72.577841</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>South Dakota</td>\n",
              "      <td>2</td>\n",
              "      <td>43.969515</td>\n",
              "      <td>-99.901813</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>Wyoming</td>\n",
              "      <td>1</td>\n",
              "      <td>43.075968</td>\n",
              "      <td>-107.290284</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-71f9613d-f62a-4616-854c-cb3be0dc3b4c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-71f9613d-f62a-4616-854c-cb3be0dc3b4c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-71f9613d-f62a-4616-854c-cb3be0dc3b4c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "map_data.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "tpkxAFipCUyF",
        "outputId": "75e4a7dc-3717-47b2-a829-bfb2b823ffbe"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    state  cases        lat          lon\n",
              "1  Alabama     53  32.318231   -86.902298\n",
              "2   Alaska      3  63.588753  -154.493062"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-50530e02-ad24-474f-a514-9f264c910b8c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>state</th>\n",
              "      <th>cases</th>\n",
              "      <th>lat</th>\n",
              "      <th>lon</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Alabama</td>\n",
              "      <td>53</td>\n",
              "      <td>32.318231</td>\n",
              "      <td>-86.902298</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Alaska</td>\n",
              "      <td>3</td>\n",
              "      <td>63.588753</td>\n",
              "      <td>-154.493062</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-50530e02-ad24-474f-a514-9f264c910b8c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-50530e02-ad24-474f-a514-9f264c910b8c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-50530e02-ad24-474f-a514-9f264c910b8c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "map_data.to_csv('map_data.csv')  "
      ],
      "metadata": {
        "id": "GUkY6bJxr4ac"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 6 | Deployment"
      ],
      "metadata": {
        "id": "3FwMqfy5dSsM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "See draft streamlit app here: https://slp22-data-engineering-project-streamlit-mpx-app-ckpzq2.streamlitapp.com/"
      ],
      "metadata": {
        "id": "XRtWhiYxeF7u"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 7 | Testing/Robustness (if time allows)"
      ],
      "metadata": {
        "id": "L31bOQ0leG-h"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "[Python schedule](https://schedule.readthedocs.io/en/stable/examples.html#run-a-job-every-x-minute)"
      ],
      "metadata": {
        "id": "fkP73gNFeJjm"
      }
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "Oscqdm5WZ3Ap",
        "KQGIpl2swXl1",
        "gOaVANqDPdyW",
        "5WYNxkC8Vhd0",
        "o58AqssjdG6J",
        "S5iwVAVOOUlQ",
        "3FwMqfy5dSsM",
        "L31bOQ0leG-h"
      ],
      "name": "engineering_monkeypox_mvp.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (engineering)",
      "language": "python",
      "name": "engineering"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}