Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "05dec84b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import re\n",
    "import os\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "912c4276",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'text', 'location', 'place', 'country', 'retweeted',\n",
      "       'retweet_count', 'favorited', 'favorite_count', 'language', 'date',\n",
      "       'verified', 'verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,'],\n",
      "      dtype='object')\n",
      "                    id                                               text  \\\n",
      "0  1319111210864939011  RT @RepValDemings: Amy Coney Barrett ruled tha...   \n",
      "1  1319111212194549761  RT @realDonaldTrump: Just don’t see any way Na...   \n",
      "2  1319111212672667648  RT @ddale8: It turns out Trump has been making...   \n",
      "3  1319111212743815169  RT @Laurie_Garrett: This would literally amoun...   \n",
      "4  1319111212794249216  RT @JackPosobiec: I have never cursed on Twitt...   \n",
      "\n",
      "           location place country retweeted retweet_count favorited  \\\n",
      "0            Kansas   NaN     NaN     False          3450     False   \n",
      "1    Pittsburgh, PA   NaN     NaN     False         17984     False   \n",
      "2     Ridgewood, NJ   NaN     NaN     False           462     False   \n",
      "3     Virginia, USA   NaN     NaN     False          1580     False   \n",
      "4  Fort Worth Texas   NaN     NaN     False          1857     False   \n",
      "\n",
      "  favorite_count language      date verified  \\\n",
      "0              0       en  10-22-03    False   \n",
      "1              0       en  10-22-03    False   \n",
      "2              0       en  10-22-03    False   \n",
      "3              0       en  10-22-03    False   \n",
      "4              0       en  10-22-03    False   \n",
      "\n",
      "  verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,  \n",
      "0                                  NaN  \n",
      "1                                  NaN  \n",
      "2                                  NaN  \n",
      "3                                  NaN  \n",
      "4                                  NaN  \n"
     ]
    }
   ],
   "source": [
    "# ignore warnings for tweets incorrectly parsed due to them containing ;\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "df = None\n",
    "folder = 'hydrated_tweets/'\n",
    "for filename in os.listdir(folder):\n",
    "    if filename == '.DS_Store':\n",
    "            continue\n",
    "    newname = folder+filename\n",
    "    for name in os.listdir(newname):\n",
    "        if name == '.DS_Store':\n",
    "            continue\n",
    "        f = os.path.join(newname, name)\n",
    "        if df is None:\n",
    "            df = pd.read_csv(f, sep=';', engine='python',quotechar='\"', error_bad_lines=False,warn_bad_lines=False)\n",
    "        else:\n",
    "            tempdf = pd.read_csv(f, sep=';',engine='python',quotechar='\"', error_bad_lines=False,warn_bad_lines=False)\n",
    "            df = pd.concat([df,tempdf], ignore_index=True)\n",
    "\n",
    "print(df.keys())\n",
    "print(df.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9ed492c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(f'alltweets.csv',sep=';',index=False,header=True,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d2ab6abe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>location</th>\n",
       "      <th>place</th>\n",
       "      <th>country</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorited</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>language</th>\n",
       "      <th>date</th>\n",
       "      <th>verified</th>\n",
       "      <th>verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1319111210864939011</td>\n",
       "      <td>RT @RepValDemings: Amy Coney Barrett ruled tha...</td>\n",
       "      <td>Kansas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>3450</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1319111212194549761</td>\n",
       "      <td>RT @realDonaldTrump: Just don’t see any way Na...</td>\n",
       "      <td>Pittsburgh, PA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>17984</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1319111212672667648</td>\n",
       "      <td>RT @ddale8: It turns out Trump has been making...</td>\n",
       "      <td>Ridgewood, NJ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>462</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1319111212743815169</td>\n",
       "      <td>RT @Laurie_Garrett: This would literally amoun...</td>\n",
       "      <td>Virginia, USA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>1580</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1319111212794249216</td>\n",
       "      <td>RT @JackPosobiec: I have never cursed on Twitt...</td>\n",
       "      <td>Fort Worth Texas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>1857</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id                                               text  \\\n",
       "0  1319111210864939011  RT @RepValDemings: Amy Coney Barrett ruled tha...   \n",
       "1  1319111212194549761  RT @realDonaldTrump: Just don’t see any way Na...   \n",
       "2  1319111212672667648  RT @ddale8: It turns out Trump has been making...   \n",
       "3  1319111212743815169  RT @Laurie_Garrett: This would literally amoun...   \n",
       "4  1319111212794249216  RT @JackPosobiec: I have never cursed on Twitt...   \n",
       "\n",
       "           location place country retweeted retweet_count favorited  \\\n",
       "0            Kansas   NaN     NaN     False          3450     False   \n",
       "1    Pittsburgh, PA   NaN     NaN     False         17984     False   \n",
       "2     Ridgewood, NJ   NaN     NaN     False           462     False   \n",
       "3     Virginia, USA   NaN     NaN     False          1580     False   \n",
       "4  Fort Worth Texas   NaN     NaN     False          1857     False   \n",
       "\n",
       "  favorite_count language      date verified  \\\n",
       "0              0       en  10-22-03    False   \n",
       "1              0       en  10-22-03    False   \n",
       "2              0       en  10-22-03    False   \n",
       "3              0       en  10-22-03    False   \n",
       "4              0       en  10-22-03    False   \n",
       "\n",
       "  verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,  \n",
       "0                                  NaN  \n",
       "1                                  NaN  \n",
       "2                                  NaN  \n",
       "3                                  NaN  \n",
       "4                                  NaN  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#alltweetsdf = pd.read_csv('alltweets.csv',sep=';')\n",
    "alltweetsdf = df\n",
    "alltweetsdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "bfdd95c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22305097, 13)\n",
      "id                                             3\n",
      "text                                       40927\n",
      "location                                  100808\n",
      "place                                   21995509\n",
      "country                                 21992149\n",
      "retweeted                                  41359\n",
      "retweet_count                              41359\n",
      "favorited                                  41359\n",
      "favorite_count                             41359\n",
      "language                                   41359\n",
      "date                                       41363\n",
      "verified                                  134637\n",
      "verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,    22215236\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(alltweetsdf.shape)\n",
    "print(alltweetsdf.isnull().sum(axis = 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8a166a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove incorrectly parsed rows and tweets which aren't english\n",
    "alltweetsdf = alltweetsdf.drop(alltweetsdf[(alltweetsdf['verified'].isnull()==True)].index)\n",
    "alltweetsdf = alltweetsdf.drop(alltweetsdf[(alltweetsdf['text'].isnull()==True)].index)\n",
    "alltweetsdf = alltweetsdf.drop(alltweetsdf[(alltweetsdf['language']!='en')].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4bef3f79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# dropping certain columns\n",
    "print((alltweetsdf['retweeted']==True).sum(axis = 0))\n",
    "print((alltweetsdf['favorited']==True).sum(axis = 0))\n",
    "alltweetsdf = alltweetsdf.drop(columns=['retweeted','favorited','language','verified,,,,,,,,,,,,,,,,,,,,,,,,,,,,'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "02a5f1e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19303737, 9)\n",
      "id                       0\n",
      "text                     0\n",
      "location             43067\n",
      "place             19066901\n",
      "country           19066921\n",
      "retweet_count            0\n",
      "favorite_count           0\n",
      "date                     0\n",
      "verified                 0\n",
      "dtype: int64\n",
      "Retweets: 12291948\n"
     ]
    }
   ],
   "source": [
    "print(alltweetsdf.shape)\n",
    "print(alltweetsdf.isnull().sum(axis = 0))\n",
    "# bit more than half the tweets are retweets\n",
    "print('Retweets:',alltweetsdf['text'].duplicated().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "0fa86f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "states = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware', 'Florida', 'Georgia', 'Hawaii', 'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky', 'Louisiana', 'Maine', 'Maryland', 'Massachusetts', 'Michigan', 'Minnesota', 'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada', 'New Hampshire', 'New Jersey', 'New York', 'New Mexico', 'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania', 'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virginia', 'Washington', 'West Virginia', 'Wisconsin', 'Wyoming']\n",
    "stateCodes = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']\n",
    "stateMapping = {'AL': 'Alabama', 'AK': 'Alaska', 'AZ': 'Arizona', 'AR': 'Arkansas', 'CA': 'California', 'CO': 'Colorado', 'CT': 'Connecticut', 'DE': 'Delaware', 'FL': 'Florida', 'GA': 'Georgia', 'HI': 'Hawaii', 'ID': 'Idaho', 'IL': 'Illinois', 'IN': 'Indiana', 'IA': 'Iowa', 'KS': 'Kansas', 'KY': 'Kentucky', 'LA': 'Louisiana', 'ME': 'Maine', 'MD': 'Maryland', 'MA': 'Massachusetts', 'MI': 'Michigan', 'MN': 'Minnesota', 'MS': 'Mississippi', 'MO': 'Missouri', 'MT': 'Montana', 'NE': 'Nebraska', 'NV': 'Nevada', 'NH': 'New Hampshire', 'NJ': 'New Jersey', 'NY': 'New York', 'NM': 'New Mexico', 'NC': 'North Carolina', 'ND': 'North Dakota', 'OH': 'Ohio', 'OK': 'Oklahoma', 'OR': 'Oregon', 'PA': 'Pennsylvania', 'RI': 'Rhode Island', 'SC': 'South Carolina', 'SD': 'South Dakota', 'TN': 'Tennessee', 'TX': 'Texas', 'UT':  'Utah', 'VT': 'Vermont', 'VA': 'Virginia', 'WA': 'Washington', 'WV':  'West Virginia', 'WI': 'Wisconsin', 'WY': 'Wyoming','D.C.':'District of Columbia'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "ad2dd5d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# some people have multiple states in their description, first state check is their given state\n",
    "def findstate(x):\n",
    "    location = x[0]\n",
    "    place = x[1]\n",
    "    for state in stateMapping:\n",
    "        # seperate if statement for DC to check for multiple ways people write DC\n",
    "        if state == 'D.C.':\n",
    "            if f', {state}' in str(location) or stateMapping[state] in str(location) or f', {state}' in str(place) or stateMapping[state] in str(place) or ', DC' in str(location) or ', DC' in str(place):\n",
    "                return stateMapping[state]\n",
    "        elif f', {state}' in str(location) or stateMapping[state] in str(location) or f', {state}' in str(place) or stateMapping[state] in str(place):\n",
    "            # checks that washington DC is not read as the state washington\n",
    "            if state == 'WA' and 'D.C.' in str(location) or 'D.C.' in str(place) or ', DC' in str(location) or ', DC' in str(place):\n",
    "                continue\n",
    "            return stateMapping[state]\n",
    "    return None\n",
    "alltweetsdf[\"state\"] = np.nan\n",
    "alltweetsdf[\"state\"] = alltweetsdf.apply(lambda row : findstate([row['location'],row['place']]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ecf80f3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>location</th>\n",
       "      <th>place</th>\n",
       "      <th>country</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>date</th>\n",
       "      <th>verified</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1319111210864939011</td>\n",
       "      <td>RT @RepValDemings: Amy Coney Barrett ruled tha...</td>\n",
       "      <td>Kansas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3450</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Kansas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1319111212194549761</td>\n",
       "      <td>RT @realDonaldTrump: Just don’t see any way Na...</td>\n",
       "      <td>Pittsburgh, PA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17984</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Pennsylvania</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1319111212672667648</td>\n",
       "      <td>RT @ddale8: It turns out Trump has been making...</td>\n",
       "      <td>Ridgewood, NJ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>462</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>New Jersey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1319111212743815169</td>\n",
       "      <td>RT @Laurie_Garrett: This would literally amoun...</td>\n",
       "      <td>Virginia, USA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Virginia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1319111212794249216</td>\n",
       "      <td>RT @JackPosobiec: I have never cursed on Twitt...</td>\n",
       "      <td>Fort Worth Texas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1857</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1319111212831887360</td>\n",
       "      <td>RT @Amy_Siskind: Note every bit of foreign int...</td>\n",
       "      <td>Colorado</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>694</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Colorado</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1319111212853022721</td>\n",
       "      <td>RT @donwinslow: Hunter Biden is not responsibl...</td>\n",
       "      <td>Boston, MA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5215</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Massachusetts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1319111213087924226</td>\n",
       "      <td>RT @willchamberlain: Maybe *you* never lived u...</td>\n",
       "      <td>One-Party-Rule California</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>602</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1319111213222141952</td>\n",
       "      <td>RT @zachsilberberg: what a difficult week for ...</td>\n",
       "      <td>Toronto, Ontario</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>586</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1319111213238964226</td>\n",
       "      <td>RT @TCRG2012: @cynthia_ann @LisaWNC1 @Kelleyro...</td>\n",
       "      <td>Western Massachusetts</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Massachusetts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1319111213368881153</td>\n",
       "      <td>#SoeakUpVoteBiden \\nI voted Biden at my clerk'...</td>\n",
       "      <td>Michigan, USA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Michigan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1319111213456949248</td>\n",
       "      <td>RT @iaingrae: @CanzyD @funder @sebastianbach E...</td>\n",
       "      <td>✝💞 Holy Spirit leads....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1319111213557624833</td>\n",
       "      <td>RT @SteveSchmidtSES: .@RudyGiuliani “Everythin...</td>\n",
       "      <td>On The Bus</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3930</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1319111213708570624</td>\n",
       "      <td>@bebe_cubana @bookkeepPLUS @ColbyCovMMA @realD...</td>\n",
       "      <td>Enumclaw Wa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1319111215214333952</td>\n",
       "      <td>RT @SenKamalaHarris: Justice Ruth Bader Ginsbu...</td>\n",
       "      <td>Milton, Ontario</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3267</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1319111216799846405</td>\n",
       "      <td>RT @tedlieu: Dear @ODNIgov Ratcliffe: Actually...</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8978</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>New York</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1319111216845983747</td>\n",
       "      <td>RT @dennisdiclaudio: My dog has been punished ...</td>\n",
       "      <td>NYC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9299</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1319111216950878215</td>\n",
       "      <td>RT @ChuckRossDC: Iran attacked an obvious weak...</td>\n",
       "      <td>Clown World Prison Planet</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>666</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1319111217064087553</td>\n",
       "      <td>RT @realDonaldTrump: THANK YOU NORTH CAROLINA!...</td>\n",
       "      <td>earth</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6576</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1319111217236135936</td>\n",
       "      <td>RT @ProfMJCleveland: Wow! https://t.co/hCOd0W7...</td>\n",
       "      <td>east TEXAS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1319111217332416512</td>\n",
       "      <td>RT @JackPosobiec: The FBI just confirmed Iran ...</td>\n",
       "      <td>Las Vegas, NV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6779</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Nevada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1319111217466802177</td>\n",
       "      <td>RT @DrEricDing: HORRIFYING—White House is cons...</td>\n",
       "      <td>Wrentham, MA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>676</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Massachusetts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1319111217831526400</td>\n",
       "      <td>RT @JackPosobiec: Delete your account you anti...</td>\n",
       "      <td>Portland, OR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1552</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Oregon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1319111217835814912</td>\n",
       "      <td>RT @KamalaHarris: Why​ would an incumbent pres...</td>\n",
       "      <td>Someplace on Earth...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25472</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1319111217806528512</td>\n",
       "      <td>@realDonaldTrump In 4 years you have enriched ...</td>\n",
       "      <td>Hamilton, Ontario</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1319111217949122562</td>\n",
       "      <td>RT @donwinslow: MY NEW VIDEO:  #NoOneWantsYour...</td>\n",
       "      <td>Otherworld</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15842</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1319111219299504130</td>\n",
       "      <td>RT @CoryBooker: Please, please help if you can...</td>\n",
       "      <td>California, USA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3231</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1319111220297932802</td>\n",
       "      <td>RT @realDonaldTrump: Just don’t see any way Na...</td>\n",
       "      <td>Volusia County</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17984</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1319111221191163909</td>\n",
       "      <td>RT @jacobsoboroff: Here's audio from a convers...</td>\n",
       "      <td>Bailey, Colorado</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>842</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Colorado</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1319111221472333824</td>\n",
       "      <td>RT @kylegriffin1: Billy Graham's granddaughter...</td>\n",
       "      <td>Florida</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5967</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Florida</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1319111221522567168</td>\n",
       "      <td>RT @DavidCornDC: Tax returns, please. American...</td>\n",
       "      <td>Anaheim, California</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>920</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1319111221623312384</td>\n",
       "      <td>RT @ElieNYC: I don't know that there are two a...</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1319111221732257792</td>\n",
       "      <td>RT @GOP: “If Biden wins, China will own the Un...</td>\n",
       "      <td>Texarklama City, TA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2359</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1319111221753249796</td>\n",
       "      <td>RT @gtconway3d: Exactly.  And it's going to work.</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4598</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Arizona</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1319111221832974340</td>\n",
       "      <td>RT @IwriteOK: my favorite genre of tweet is ri...</td>\n",
       "      <td>Philadelphia, PA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>563</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Pennsylvania</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1319111221858193408</td>\n",
       "      <td>RT @CNNPolitics: Republican Sen. Mitt Romney: ...</td>\n",
       "      <td>Tampa, FL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1155</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Florida</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1319111221895962626</td>\n",
       "      <td>RT @realDonaldTrump: WSJ Editorial Board: Joe ...</td>\n",
       "      <td>Plymouth, MN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9433</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Minnesota</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1319111221958922240</td>\n",
       "      <td>RT @JackPosobiec: Biden just said America was ...</td>\n",
       "      <td>Ottawa Canada 🇨🇦</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4650</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1319111222080438272</td>\n",
       "      <td>RT @MrAndyNgo: Threatening messages sent to De...</td>\n",
       "      <td>St. Petersburg, FL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2385</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>True</td>\n",
       "      <td>Florida</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1319111222135033856</td>\n",
       "      <td>He's delusional. And everyone knows it. https:...</td>\n",
       "      <td>Third booth from the door.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1319111222156034049</td>\n",
       "      <td>RT @msleen1970: Y’all noticed how the conversa...</td>\n",
       "      <td>Houston | Nashville</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>252</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1319111222164324352</td>\n",
       "      <td>RT @KyleKulinski: This is great! Now do all of...</td>\n",
       "      <td>Your Ego</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>335</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1319111222227337218</td>\n",
       "      <td>RT @realDonaldTrump: I am not just running aga...</td>\n",
       "      <td>Jersey</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32635</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>1319111223993131009</td>\n",
       "      <td>RT @KamalaHarris: Why​ would an incumbent pres...</td>\n",
       "      <td>Austin, TX</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25472</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1319111225276616705</td>\n",
       "      <td>@fylth1 @UfoAntifa @blackvaultcom Who do you t...</td>\n",
       "      <td>Nunya Effin Binness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>1319111225553211393</td>\n",
       "      <td>RT @tedlieu: I’ve concluded this hastily arran...</td>\n",
       "      <td>Calgary, Alberta</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2647</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>1319111225632944128</td>\n",
       "      <td>RT @MrsB_PNW: @SteveScalise @realDonaldTrump W...</td>\n",
       "      <td>Chico, CA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>1319111225645494275</td>\n",
       "      <td>@JoeBiden Paedophilia is an idea. And @JoeBide...</td>\n",
       "      <td>ur mum :---D</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>1319111225804992512</td>\n",
       "      <td>@christielt @DumpTrump_45 @Goodmoringmm @Kamal...</td>\n",
       "      <td>My house</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>1319111225905610752</td>\n",
       "      <td>@DonaldJTrumpJr @realDonaldTrump I can’t wait ...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>District of Columbia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     id                                               text  \\\n",
       "0   1319111210864939011  RT @RepValDemings: Amy Coney Barrett ruled tha...   \n",
       "1   1319111212194549761  RT @realDonaldTrump: Just don’t see any way Na...   \n",
       "2   1319111212672667648  RT @ddale8: It turns out Trump has been making...   \n",
       "3   1319111212743815169  RT @Laurie_Garrett: This would literally amoun...   \n",
       "4   1319111212794249216  RT @JackPosobiec: I have never cursed on Twitt...   \n",
       "5   1319111212831887360  RT @Amy_Siskind: Note every bit of foreign int...   \n",
       "6   1319111212853022721  RT @donwinslow: Hunter Biden is not responsibl...   \n",
       "7   1319111213087924226  RT @willchamberlain: Maybe *you* never lived u...   \n",
       "8   1319111213222141952  RT @zachsilberberg: what a difficult week for ...   \n",
       "9   1319111213238964226  RT @TCRG2012: @cynthia_ann @LisaWNC1 @Kelleyro...   \n",
       "10  1319111213368881153  #SoeakUpVoteBiden \\nI voted Biden at my clerk'...   \n",
       "11  1319111213456949248  RT @iaingrae: @CanzyD @funder @sebastianbach E...   \n",
       "12  1319111213557624833  RT @SteveSchmidtSES: .@RudyGiuliani “Everythin...   \n",
       "13  1319111213708570624  @bebe_cubana @bookkeepPLUS @ColbyCovMMA @realD...   \n",
       "14  1319111215214333952  RT @SenKamalaHarris: Justice Ruth Bader Ginsbu...   \n",
       "15  1319111216799846405  RT @tedlieu: Dear @ODNIgov Ratcliffe: Actually...   \n",
       "16  1319111216845983747  RT @dennisdiclaudio: My dog has been punished ...   \n",
       "17  1319111216950878215  RT @ChuckRossDC: Iran attacked an obvious weak...   \n",
       "18  1319111217064087553  RT @realDonaldTrump: THANK YOU NORTH CAROLINA!...   \n",
       "20  1319111217236135936  RT @ProfMJCleveland: Wow! https://t.co/hCOd0W7...   \n",
       "21  1319111217332416512  RT @JackPosobiec: The FBI just confirmed Iran ...   \n",
       "22  1319111217466802177  RT @DrEricDing: HORRIFYING—White House is cons...   \n",
       "23  1319111217831526400  RT @JackPosobiec: Delete your account you anti...   \n",
       "24  1319111217835814912  RT @KamalaHarris: Why​ would an incumbent pres...   \n",
       "25  1319111217806528512  @realDonaldTrump In 4 years you have enriched ...   \n",
       "27  1319111217949122562  RT @donwinslow: MY NEW VIDEO:  #NoOneWantsYour...   \n",
       "29  1319111219299504130  RT @CoryBooker: Please, please help if you can...   \n",
       "30  1319111220297932802  RT @realDonaldTrump: Just don’t see any way Na...   \n",
       "32  1319111221191163909  RT @jacobsoboroff: Here's audio from a convers...   \n",
       "33  1319111221472333824  RT @kylegriffin1: Billy Graham's granddaughter...   \n",
       "34  1319111221522567168  RT @DavidCornDC: Tax returns, please. American...   \n",
       "36  1319111221623312384  RT @ElieNYC: I don't know that there are two a...   \n",
       "37  1319111221732257792  RT @GOP: “If Biden wins, China will own the Un...   \n",
       "38  1319111221753249796  RT @gtconway3d: Exactly.  And it's going to work.   \n",
       "39  1319111221832974340  RT @IwriteOK: my favorite genre of tweet is ri...   \n",
       "40  1319111221858193408  RT @CNNPolitics: Republican Sen. Mitt Romney: ...   \n",
       "41  1319111221895962626  RT @realDonaldTrump: WSJ Editorial Board: Joe ...   \n",
       "42  1319111221958922240  RT @JackPosobiec: Biden just said America was ...   \n",
       "44  1319111222080438272  RT @MrAndyNgo: Threatening messages sent to De...   \n",
       "45  1319111222135033856  He's delusional. And everyone knows it. https:...   \n",
       "46  1319111222156034049  RT @msleen1970: Y’all noticed how the conversa...   \n",
       "47  1319111222164324352  RT @KyleKulinski: This is great! Now do all of...   \n",
       "48  1319111222227337218  RT @realDonaldTrump: I am not just running aga...   \n",
       "49  1319111223993131009  RT @KamalaHarris: Why​ would an incumbent pres...   \n",
       "51  1319111225276616705  @fylth1 @UfoAntifa @blackvaultcom Who do you t...   \n",
       "53  1319111225553211393  RT @tedlieu: I’ve concluded this hastily arran...   \n",
       "54  1319111225632944128  RT @MrsB_PNW: @SteveScalise @realDonaldTrump W...   \n",
       "55  1319111225645494275  @JoeBiden Paedophilia is an idea. And @JoeBide...   \n",
       "56  1319111225804992512  @christielt @DumpTrump_45 @Goodmoringmm @Kamal...   \n",
       "58  1319111225905610752  @DonaldJTrumpJr @realDonaldTrump I can’t wait ...   \n",
       "\n",
       "                      location place country retweet_count favorite_count  \\\n",
       "0                       Kansas   NaN     NaN          3450              0   \n",
       "1               Pittsburgh, PA   NaN     NaN         17984              0   \n",
       "2                Ridgewood, NJ   NaN     NaN           462              0   \n",
       "3                Virginia, USA   NaN     NaN          1580              0   \n",
       "4             Fort Worth Texas   NaN     NaN          1857              0   \n",
       "5                     Colorado   NaN     NaN           694              0   \n",
       "6                   Boston, MA   NaN     NaN          5215              0   \n",
       "7   One-Party-Rule California    NaN     NaN           602              0   \n",
       "8             Toronto, Ontario   NaN     NaN           586              0   \n",
       "9        Western Massachusetts   NaN     NaN             2              0   \n",
       "10               Michigan, USA   NaN     NaN             0              2   \n",
       "11    ✝💞 Holy Spirit leads....   NaN     NaN             1              0   \n",
       "12                  On The Bus   NaN     NaN          3930              0   \n",
       "13                 Enumclaw Wa   NaN     NaN             0              0   \n",
       "14             Milton, Ontario   NaN     NaN          3267              0   \n",
       "15                New York, NY   NaN     NaN          8978              0   \n",
       "16                         NYC   NaN     NaN          9299              0   \n",
       "17   Clown World Prison Planet   NaN     NaN           666              0   \n",
       "18                       earth   NaN     NaN          6576              0   \n",
       "20                  east TEXAS   NaN     NaN           204              0   \n",
       "21               Las Vegas, NV   NaN     NaN          6779              0   \n",
       "22                Wrentham, MA   NaN     NaN           676              0   \n",
       "23                Portland, OR   NaN     NaN          1552              0   \n",
       "24       Someplace on Earth...   NaN     NaN         25472              0   \n",
       "25           Hamilton, Ontario   NaN     NaN             0              0   \n",
       "27                  Otherworld   NaN     NaN         15842              0   \n",
       "29             California, USA   NaN     NaN          3231              0   \n",
       "30              Volusia County   NaN     NaN         17984              0   \n",
       "32            Bailey, Colorado   NaN     NaN           842              0   \n",
       "33                     Florida   NaN     NaN          5967              0   \n",
       "34         Anaheim, California   NaN     NaN           920              0   \n",
       "36                 Los Angeles   NaN     NaN            76              0   \n",
       "37         Texarklama City, TA   NaN     NaN          2359              0   \n",
       "38                     Arizona   NaN     NaN          4598              0   \n",
       "39            Philadelphia, PA   NaN     NaN           563              0   \n",
       "40                   Tampa, FL   NaN     NaN          1155              0   \n",
       "41                Plymouth, MN   NaN     NaN          9433              0   \n",
       "42            Ottawa Canada 🇨🇦   NaN     NaN          4650              0   \n",
       "44          St. Petersburg, FL   NaN     NaN          2385              0   \n",
       "45  Third booth from the door.   NaN     NaN             0              0   \n",
       "46        Houston | Nashville    NaN     NaN           252              0   \n",
       "47                    Your Ego   NaN     NaN           335              0   \n",
       "48                      Jersey   NaN     NaN         32635              0   \n",
       "49                  Austin, TX   NaN     NaN         25472              0   \n",
       "51         Nunya Effin Binness   NaN     NaN             0              0   \n",
       "53            Calgary, Alberta   NaN     NaN          2647              0   \n",
       "54                   Chico, CA   NaN     NaN             4              0   \n",
       "55                ur mum :---D   NaN     NaN             0              3   \n",
       "56                    My house   NaN     NaN             0              1   \n",
       "58              Washington, DC   NaN     NaN             0              0   \n",
       "\n",
       "        date verified                 state  \n",
       "0   10-22-03    False                Kansas  \n",
       "1   10-22-03    False          Pennsylvania  \n",
       "2   10-22-03    False            New Jersey  \n",
       "3   10-22-03    False              Virginia  \n",
       "4   10-22-03    False                 Texas  \n",
       "5   10-22-03    False              Colorado  \n",
       "6   10-22-03    False         Massachusetts  \n",
       "7   10-22-03    False            California  \n",
       "8   10-22-03    False                  None  \n",
       "9   10-22-03    False         Massachusetts  \n",
       "10  10-22-03    False              Michigan  \n",
       "11  10-22-03    False                  None  \n",
       "12  10-22-03    False                  None  \n",
       "13  10-22-03    False                  None  \n",
       "14  10-22-03    False                  None  \n",
       "15  10-22-03    False              New York  \n",
       "16  10-22-03    False                  None  \n",
       "17  10-22-03    False                  None  \n",
       "18  10-22-03    False                  None  \n",
       "20  10-22-03    False                  None  \n",
       "21  10-22-03    False                Nevada  \n",
       "22  10-22-03    False         Massachusetts  \n",
       "23  10-22-03    False                Oregon  \n",
       "24  10-22-03    False                  None  \n",
       "25  10-22-03    False                  None  \n",
       "27  10-22-03    False                  None  \n",
       "29  10-22-03    False            California  \n",
       "30  10-22-03    False                  None  \n",
       "32  10-22-03    False              Colorado  \n",
       "33  10-22-03    False               Florida  \n",
       "34  10-22-03    False            California  \n",
       "36  10-22-03    False                  None  \n",
       "37  10-22-03    False                  None  \n",
       "38  10-22-03    False               Arizona  \n",
       "39  10-22-03    False          Pennsylvania  \n",
       "40  10-22-03    False               Florida  \n",
       "41  10-22-03    False             Minnesota  \n",
       "42  10-22-03    False                  None  \n",
       "44  10-22-03     True               Florida  \n",
       "45  10-22-03    False                  None  \n",
       "46  10-22-03    False                  None  \n",
       "47  10-22-03    False                  None  \n",
       "48  10-22-03    False                  None  \n",
       "49  10-22-03    False                 Texas  \n",
       "51  10-22-03    False                  None  \n",
       "53  10-22-03    False                  None  \n",
       "54  10-22-03    False            California  \n",
       "55  10-22-03    False                  None  \n",
       "56  10-22-03    False                  None  \n",
       "58  10-22-03    False  District of Columbia  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alltweetsdf.head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "096c78dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19303737, 10)\n",
      "id                       0\n",
      "text                     0\n",
      "location             43067\n",
      "place             19066901\n",
      "country           19066921\n",
      "retweet_count            0\n",
      "favorite_count           0\n",
      "date                     0\n",
      "verified                 0\n",
      "state              9848184\n",
      "dtype: int64\n",
      "Retweets: 12291948\n"
     ]
    }
   ],
   "source": [
    "print(alltweetsdf.shape)\n",
    "print(alltweetsdf.isnull().sum(axis = 0))\n",
    "print('Retweets:',alltweetsdf['text'].duplicated().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c9f177c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete nan rows from states\n",
    "# delete location place and country columns\n",
    "alltweetsdf = alltweetsdf.drop(alltweetsdf[(alltweetsdf['state'].isnull()==True)].index)\n",
    "alltweetsdf = alltweetsdf.drop(columns=['location','place','country'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "3d84c5d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9455553, 7)\n",
      "id                0\n",
      "text              0\n",
      "retweet_count     0\n",
      "favorite_count    0\n",
      "date              0\n",
      "verified          0\n",
      "state             0\n",
      "dtype: int64\n",
      "Retweets: 5919982\n"
     ]
    }
   ],
   "source": [
    "print(alltweetsdf.shape)\n",
    "print(alltweetsdf.isnull().sum(axis = 0))\n",
    "print('Retweets:',alltweetsdf['text'].duplicated().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "e6d573de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "California              1312674\n",
       "Texas                    936219\n",
       "Florida                  835746\n",
       "New York                 666869\n",
       "Pennsylvania             331117\n",
       "Ohio                     297217\n",
       "Illinois                 282975\n",
       "Washington               269193\n",
       "Arizona                  265532\n",
       "Georgia                  264461\n",
       "Michigan                 261664\n",
       "North Carolina           246036\n",
       "Virginia                 241849\n",
       "New Jersey               208442\n",
       "Massachusetts            208283\n",
       "Colorado                 197602\n",
       "Oregon                   182661\n",
       "Tennessee                169081\n",
       "District of Columbia     167601\n",
       "Maryland                 151955\n",
       "Minnesota                148411\n",
       "Indiana                  135123\n",
       "Wisconsin                123221\n",
       "South Carolina           114771\n",
       "Nevada                   111810\n",
       "Missouri                 107252\n",
       "Alabama                  103370\n",
       "Kentucky                  95182\n",
       "Louisiana                 92316\n",
       "Oklahoma                  86047\n",
       "Kansas                    84889\n",
       "Connecticut               83967\n",
       "Arkansas                  60952\n",
       "Iowa                      59321\n",
       "Utah                      58167\n",
       "New Mexico                52551\n",
       "Hawaii                    49110\n",
       "Maine                     45134\n",
       "Nebraska                  41374\n",
       "New Hampshire             39503\n",
       "Mississippi               39494\n",
       "Idaho                     38156\n",
       "Delaware                  31349\n",
       "Montana                   30309\n",
       "Alaska                    28045\n",
       "Rhode Island              26956\n",
       "Vermont                   21332\n",
       "South Dakota              16194\n",
       "West Virginia             12220\n",
       "Wyoming                   12133\n",
       "North Dakota               9717\n",
       "Name: state, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alltweetsdf['state'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "be156bad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Word  Frequency\n",
      "0     RT    6685883\n",
      "1    the    5317061\n",
      "2     to    4174267\n",
      "3      a    2847701\n",
      "4    and    2709938\n",
      "5     of    2498969\n",
      "6     is    2466279\n",
      "7     in    2435311\n",
      "8  Trump    1986047\n",
      "9    for    1835860\n"
     ]
    }
   ],
   "source": [
    "word_count = Counter(\" \".join(alltweetsdf['text']).split()).most_common(10)\n",
    "word_frequency = pd.DataFrame(word_count, columns = ['Word', 'Frequency'])\n",
    "print(word_frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "4ce9e9c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = [\"for\", \"on\", \"an\", \"a\", \"of\", \"and\", \"in\", \"the\", \"to\", \"from\", \"rt\", 'this','that','is','are','you','i','it','we','be','have']\n",
    "\n",
    "def clean_tweet(tweet):\n",
    "    temp = tweet.lower()\n",
    "    temp = re.sub(r'http\\S+', '', temp) # removes links\n",
    "    temp = re.sub('[().,:!?]', ' ', temp)  # removes punctutation\n",
    "    temp = temp.split()\n",
    "    temp = [w for w in temp if not w in stopwords] # removes stopwords\n",
    "    temp = \" \".join(word for word in temp)\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "a13cf7bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleantweets = alltweetsdf\n",
    "cleantweets[\"filtered_text\"] = np.nan\n",
    "cleantweets[\"filtered_text\"] = cleantweets.apply(lambda row : clean_tweet(row['text']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "6669841c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>date</th>\n",
       "      <th>verified</th>\n",
       "      <th>state</th>\n",
       "      <th>filtered_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1319111210864939011</td>\n",
       "      <td>RT @RepValDemings: Amy Coney Barrett ruled tha...</td>\n",
       "      <td>3450</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Kansas</td>\n",
       "      <td>@repvaldemings amy coney barrett ruled supervi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1319111212194549761</td>\n",
       "      <td>RT @realDonaldTrump: Just don’t see any way Na...</td>\n",
       "      <td>17984</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>@realdonaldtrump just don’t see any way nancy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1319111212672667648</td>\n",
       "      <td>RT @ddale8: It turns out Trump has been making...</td>\n",
       "      <td>462</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>@ddale8 turns out trump has been making false ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1319111212743815169</td>\n",
       "      <td>RT @Laurie_Garrett: This would literally amoun...</td>\n",
       "      <td>1580</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>@laurie_garrett would literally amount partisa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1319111212794249216</td>\n",
       "      <td>RT @JackPosobiec: I have never cursed on Twitt...</td>\n",
       "      <td>1857</td>\n",
       "      <td>0</td>\n",
       "      <td>10-22-03</td>\n",
       "      <td>False</td>\n",
       "      <td>Texas</td>\n",
       "      <td>@jackposobiec never cursed twitter before but ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id                                               text  \\\n",
       "0  1319111210864939011  RT @RepValDemings: Amy Coney Barrett ruled tha...   \n",
       "1  1319111212194549761  RT @realDonaldTrump: Just don’t see any way Na...   \n",
       "2  1319111212672667648  RT @ddale8: It turns out Trump has been making...   \n",
       "3  1319111212743815169  RT @Laurie_Garrett: This would literally amoun...   \n",
       "4  1319111212794249216  RT @JackPosobiec: I have never cursed on Twitt...   \n",
       "\n",
       "  retweet_count favorite_count      date verified         state  \\\n",
       "0          3450              0  10-22-03    False        Kansas   \n",
       "1         17984              0  10-22-03    False  Pennsylvania   \n",
       "2           462              0  10-22-03    False    New Jersey   \n",
       "3          1580              0  10-22-03    False      Virginia   \n",
       "4          1857              0  10-22-03    False         Texas   \n",
       "\n",
       "                                       filtered_text  \n",
       "0  @repvaldemings amy coney barrett ruled supervi...  \n",
       "1  @realdonaldtrump just don’t see any way nancy ...  \n",
       "2  @ddale8 turns out trump has been making false ...  \n",
       "3  @laurie_garrett would literally amount partisa...  \n",
       "4  @jackposobiec never cursed twitter before but ...  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleantweets.to_csv(f'cleantweets.csv',sep=';',index=False,header=True,encoding='utf-8')\n",
    "cleantweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a059c4ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               Word  Frequency\n",
      "0             trump    2584138\n",
      "1  @realdonaldtrump    1524905\n",
      "2             biden    1506394\n",
      "3              vote     955524\n",
      "4         @joebiden     954235\n",
      "5                he     953744\n",
      "6               not     931665\n",
      "7              with     791699\n",
      "8              will     787633\n",
      "9                if     782228\n"
     ]
    }
   ],
   "source": [
    "word_count = Counter(\" \".join(cleantweets['filtered_text']).split()).most_common(10)\n",
    "word_frequency = pd.DataFrame(word_count, columns = ['Word', 'Frequency'])\n",
    "print(word_frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d34c6c4b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}