Data Preprocessing for NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "330e8cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "dl_manager = datasets.DownloadManager()\n",
    "_DOWNLOAD_URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00332/OnlineNewsPopularity.zip'\n",
    "archive = dl_manager.download(_DOWNLOAD_URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7121cb17",
   "metadata": {},
   "outputs": [],
   "source": [
    "for path, f in dl_manager.iter_archive(archive):\n",
    "    if path[-3:] == 'csv':\n",
    "        df = pd.read_csv(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "15481725",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = df.columns.str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "99fbe343",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWKUlEQVR4nO3df4zc9Z3f8efbszZrzI+a4ETBCzWnoMvAtmnC1nA969S9VJj0qoO2ic5G19jNNFYjOqXVVQS6UmlDXc40anrQBgnVriF3DAEaEXqcj1gwJ3UbCpgkdxj2XKyYH4tpMLJFwS32ev3uH/vd9axrvp41nh0PPB/SaGbe8/185j2RyWu/38/3OxOZiSRJH2RBtxuQJJ3ZDApJUimDQpJUyqCQJJUyKCRJpfq63cDpduGFF+aKFSu63YYk9ZTnn3/+7cxcdqLXPnJBsWLFCnbs2NHtNiSpp0TEqx/0moeeJEmlDApJUimDQpJUyqCQJJUyKCRJpQwKaR40Gg0GBwepVCoMDg7SaDS63ZLUto/c6bHSmabRaDAyMsLmzZtZtWoVo6Oj1Go1ANauXdvl7qSTi4/a14wPDQ2l11HoTDI4OMjdd9/N8PDwTK3ZbFKv19m5c2cXO5OOiYjnM3PohK8ZFFJnVSoV3n//fRYuXDhTm5iYoL+/n8nJyS52Jh1TFhSuUUgdVq1WGR0dnVUbHR2lWq12qSNpbgwKqcNGRkao1Wo0m00mJiZoNpvUajVGRka63ZrUFhezpQ6bXrCu1+uMjY1RrVbZuHGjC9nqGa5RSJJco5AknTqDQpJUqq2giIh/GhEvRsTOiGhERH9EXBAR2yPi5eJ+acv2t0bE7ojYFRGrW+pXRsQLxWt3RUQU9bMi4vtF/ZmIWNEyZl3xHi9HxLrT+NklSW04aVBExHLgHwNDmTkIVIA1wC3Ak5l5GfBk8ZyIuLx4/QrgWuC7EVEpprsH2ABcVtyuLeo14EBmfgb4DrCpmOsC4DbgKmAlcFtrIEmSOq/dQ099wOKI6APOBvYC1wH3Fa/fB1xfPL4OeDAzD2XmHmA3sDIiPg2cl5lP59QK+v3HjZme6xHgi8Xexmpge2buz8wDwHaOhYskaR6cNCgy8w3g28BrwJvAO5n5I+BTmflmsc2bwCeLIcuB11umGC9qy4vHx9dnjcnMI8A7wCdK5polIjZExI6I2LFv376TfSRJ0hy0c+hpKVN/8V8KXAQsiYjfLhtyglqW1E91zLFC5r2ZOZSZQ8uWnfC3wSVJp6idQ09/A9iTmfsycwL4AfDXgF8Uh5Mo7t8qth8HLm4ZP8DUoarx4vHx9VljisNb5wP7S+aSJM2TdoLiNeDqiDi7WDf4IjAGPAZMn4W0Dvhh8fgxYE1xJtOlTC1aP1scnno3Iq4u5vnqcWOm5/oy8FSxjvEEcE1ELC32bK4papKkeXLSr/DIzGci4hHgJ8AR4KfAvcA5wEMRUWMqTL5SbP9iRDwEvFRsf2NmTn9F5jeArcBiYFtxA9gMfC8idjO1J7GmmGt/RNwOPFds963M3P+hPrEkaU78Cg9Jkl/hIUk6dQaFJKmUQSFJKmVQSJJKGRSSpFIGhSSplEEhSSplUEiSShkUkqRSBoUkqZRBIUkqZVBIkkoZFJKkUgaFJKmUQSFJKmVQSJJKGRSSpFIGhTQPGo0Gg4ODVCoVBgcHaTQa3W5JattJfzNb0ofTaDQYGRlh8+bNrFq1itHRUWq1GgBr167tcnfSyfmb2VKHDQ4OcvfddzM8PDxTazab1Ot1du7c2cXOpGPKfjPboJA6rFKp8P7777Nw4cKZ2sTEBP39/UxOTnaxM+mYsqBwjULqsGq1yujo6Kza6Ogo1Wq1Sx1Jc2NQSB02MjJCrVaj2WwyMTFBs9mkVqsxMjLS7daktriYLXXY2rVr+fGPf8yXvvQlDh06xFlnncXXv/51F7LVM9yjkDqs0Wjw+OOPs23bNg4fPsy2bdt4/PHHPUVWPcPFbKnDPOtJvcDFbKmLxsbGGB8fn3XB3fj4OGNjY91uTWqLaxRSh1100UXcfPPNPPDAAzMX3N1www1cdNFF3W5Naot7FNI8iIjS59KZzKCQOmzv3r1s2rSJer1Of38/9XqdTZs2sXfv3m63JrXFQ09Sh1WrVQYGBmYtXDebTS+4U89wj0LqMC+4U69zj0LqsOkL6+r1OmNjY1SrVTZu3OgFd+oZXkchSfI6Cqnb/OEi9TIPPUkd5g8Xqdd56EnqML/CQ73AHy6SusgfLlIv+NBrFBHxFyLikYj484gYi4hfiYgLImJ7RLxc3C9t2f7WiNgdEbsiYnVL/cqIeKF47a4oLk+NiLMi4vtF/ZmIWNEyZl3xHi9HxLpT/l9B6hJ/uEi9rt3F7N8D/jgzPwt8DhgDbgGezMzLgCeL50TE5cAa4ArgWuC7EVEp5rkH2ABcVtyuLeo14EBmfgb4DrCpmOsC4DbgKmAlcFtrIEm9wOso1OtOupgdEecBvwasB8jMw8DhiLgO+OvFZvcBfwJ8E7gOeDAzDwF7ImI3sDIiXgHOy8yni3nvB64HthVj/mUx1yPAfyj2NlYD2zNzfzFmO1Ph4ikj6hleR6Fe185ZT78E7AP+c0R8DngeuAn4VGa+CZCZb0bEJ4vtlwP/o2X8eFGbKB4fX58e83ox15GIeAf4RGv9BGNmRMQGpvZUuOSSS9r4SNL8Wrt2rcGgntXOoac+4AvAPZn5eeAgxWGmD3Cir8XMkvqpjjlWyLw3M4cyc2jZsmUlrUmS5qqdoBgHxjPzmeL5I0wFxy8i4tMAxf1bLdtf3DJ+ANhb1AdOUJ81JiL6gPOB/SVzSZLmyUmDIjP/F/B6RPxyUfoi8BLwGDB9FtI64IfF48eANcWZTJcytWj9bHGY6t2IuLpYf/jqcWOm5/oy8FROnbf7BHBNRCwtFrGvKWqSpHnS7pXZdeAPImIR8HPg7zMVMg9FRA14DfgKQGa+GBEPMRUmR4AbM3P6ZPFvAFuBxUwtYm8r6puB7xUL3/uZOmuKzNwfEbcDzxXbfWt6YVuSND+84E6S5JcCSpJOnUEhSSplUEiSShkUkqRSBoUkqZRBIUkqZVBIkkoZFJKkUgaFJKmUQSFJKmVQSJJKGRSSpFIGhSSplEEhSSplUEiSShkUkqRSBoUkqZRBIUkqZVBIkkoZFJKkUgaFJKmUQSFJKmVQSJJKGRSSpFIGhSSplEEhSSplUEiSShkUkqRSBoUkqZRBIc2DRqPB4OAglUqFwcFBGo1Gt1uS2tbX7Qakj7pGo8HIyAibN29m1apVjI6OUqvVAFi7dm2Xu5NOLjKz2z2cVkNDQ7ljx45utyHNGBwc5O6772Z4eHim1mw2qdfr7Ny5s4udScdExPOZOXTC1wwKqbMqlQrvv/8+CxcunKlNTEzQ39/P5ORkFzuTjikLCtcopA6rVquMjo7Oqo2OjlKtVrvUkTQ3BoXUYSMjI9RqNZrNJhMTEzSbTWq1GiMjI91uTWqLi9lSh00vWNfrdcbGxqhWq2zcuNGFbPUM1ygkSa5RSJJOXdtBERGViPhpRPxh8fyCiNgeES8X90tbtr01InZHxK6IWN1SvzIiXiheuysioqifFRHfL+rPRMSKljHrivd4OSLWnZZPLUlq21z2KG4Cxlqe3wI8mZmXAU8Wz4mIy4E1wBXAtcB3I6JSjLkH2ABcVtyuLeo14EBmfgb4DrCpmOsC4DbgKmAlcFtrIEmSOq+toIiIAeA3gP/UUr4OuK94fB9wfUv9wcw8lJl7gN3Ayoj4NHBeZj6dUwsj9x83ZnquR4AvFnsbq4Htmbk/Mw8A2zkWLpKkedDuHsW/B24GjrbUPpWZbwIU958s6suB11u2Gy9qy4vHx9dnjcnMI8A7wCdK5polIjZExI6I2LFv3742P5IkqR0nDYqI+FvAW5n5fJtzxglqWVI/1THHCpn3ZuZQZg4tW7aszTYlSe1oZ4/iV4HfjIhXgAeBX4+I3wd+URxOorh/q9h+HLi4ZfwAsLeoD5ygPmtMRPQB5wP7S+aSJM2TkwZFZt6amQOZuYKpReqnMvO3gceA6bOQ1gE/LB4/BqwpzmS6lKlF62eLw1PvRsTVxfrDV48bMz3Xl4v3SOAJ4JqIWFosYl9T1CRJ8+TDXJn9u8BDEVEDXgO+ApCZL0bEQ8BLwBHgxsyc/uazbwBbgcXAtuIGsBn4XkTsZmpPYk0x1/6IuB14rtjuW5m5/0P0LEmaI6/MliR5ZbYk6dQZFJKkUgaFJKmUQSFJKmVQSJJKGRSSpFIGhSSplEEhSSplUEiSShkUkqRSBoUkqZRBIUkqZVBI86DRaDA4OEilUmFwcJBGo9HtlqS2GRRShzUaDW666SYOHjxIZnLw4EFuuukmw0I9w6CQOuzmm2+mUqmwZcsWDh06xJYtW6hUKtx8883dbk1qi0Ehddj4+Djr16+nXq/T399PvV5n/fr1jI+Pd7s1qS0f5hfuJLVp69atPPDAA6xatYrR0VFuuOGGbrcktc09CqnD+vr6OHz48Kza4cOH6evz7zT1Bv+lSh02OTnJxMQEq1evZmJigoULF9Lf38/k5OTJB0tnAPcopA5bvnw5lUqF5cuXExGznku9wKCQ5kF/f/+ss576+/u73ZLUNoNC6rC9e/dy5513zjrr6c4772Tv3r3dbk1qi2sUUodVq1UGBgbYuXPnTK3ZbFKtVrvYldQ+9yikDhsZGaFWq9FsNpmYmKDZbFKr1RgZGel2a1Jb3KOQOmzt2rUA1Ot1xsbGqFarbNy4caYunekiM7vdw2k1NDSUO3bs6HYbktRTIuL5zBw60WseepIklTIoJEmlDApJUimDQpJUyqCQ5oG/cKde5umxUoc1Gg1GRkbYvHnzzNeM12o1AE+RVU/w9FipwwYHB7n++ut59NFHZ66jmH7eerW21E1lp8e6RyF12EsvvcTBgwfZsmXLzB7F1772NV599dVutya1xTUKqcMWLVpEvV5neHiYhQsXMjw8TL1eZ9GiRd1uTWqLh56kDluwYAEXXnghS5Ys4bXXXuOSSy7h4MGDvP322xw9erTb7UmAh56krlq+fDlvvfUW+/btA+CVV15h0aJF/nCReoaHnqQOO3DgAIcPH2bp0qVEBEuXLuXw4cMcOHCg261JbTlpUETExRHRjIixiHgxIm4q6hdExPaIeLm4X9oy5taI2B0RuyJidUv9yoh4oXjtroiIon5WRHy/qD8TEStaxqwr3uPliFh3Wj+9NA8OHjzIokWLeO+998hM3nvvPRYtWsTBgwe73ZrUlnb2KI4Av5OZVeBq4MaIuBy4BXgyMy8DniyeU7y2BrgCuBb4bkRUirnuATYAlxW3a4t6DTiQmZ8BvgNsKua6ALgNuApYCdzWGkhSryj+JvrA59KZ7KRBkZlvZuZPisfvAmPAcuA64L5is/uA64vH1wEPZuahzNwD7AZWRsSngfMy8+mcWkG//7gx03M9Anyx2NtYDWzPzP2ZeQDYzrFwkXrGoUOHOOecc4gIzjnnHA4dOtTtlqS2zWmNojgk9HngGeBTmfkmTIUJ8Mlis+XA6y3Dxova8uLx8fVZYzLzCPAO8ImSuY7va0NE7IiIHdMLhtKZyD0J9aK2gyIizgH+C/BPMvN/l216glqW1E91zLFC5r2ZOZSZQ8uWLStpTeqO6TWKo0ePzqxRSL2iraCIiIVMhcQfZOYPivIvisNJFPdvFfVx4OKW4QPA3qI+cIL6rDER0QecD+wvmUvqKa5RqJe1c9ZTAJuBscz8dy0vPQZMn4W0DvhhS31NcSbTpUwtWj9bHJ56NyKuLub86nFjpuf6MvBUsY7xBHBNRCwtFrGvKWpSz1iyZMnMGsWCBQtm1iiWLFnS7daktrRzwd2vAn8PeCEiflbU/jnwu8BDEVEDXgO+ApCZL0bEQ8BLTJ0xdWNmThbjvgFsBRYD24obTAXR9yJiN1N7EmuKufZHxO3Ac8V238rM/af2UaXuWLp06cxpsdOHns4++2yWLvUEPvUGv8JD6rBKpcLWrVvZtGnTzLfHfvOb32T9+vVMTk6efAJpHpR9hYdXZksdVq1W2bVr16zarl27qFarXepImhuDQuqw4eFh7rjjjpkvAXz77be54447GB4e7nZrUlsMCqnDHn30Uc4991wWL17MggULWLx4Meeeey6PPvpot1uT2mJQSB02Pj7Oww8/zJ49e5icnGTPnj08/PDDjI+Pn3ywdAbwa8alebBhwwZeeeWVmecrVqzoWi/SXLlHIXVYX1/frJCAqd+k6Ovz7zT1BoNC6rAjR44AzFqjaK1LZzqDQpoHAwMDHD16dOY2MDBw8kHSGcJ9X2kevPHGG0xf3Hro0CHeeOONLncktc89CmkeZObMmkRfXx8ftW9E0EebQSHNk+k1Cdcm1GsMCklSKYNCklTKoJAklTIoJEmlDApJUimDQpJUyqCQJJUyKCRJpQwKSVIpg0KSVMqgkCSVMigkSaUMCklSKYNCklTKoJAklTIoJEmlDApJUimDQpJUyqCQJJUyKCRJpQwKSVIpg0KSVMqgkCSVMigkSaUMCklSKYNCklTKoJAkleqJoIiIayNiV0Tsjohbut2PJH2cnPFBEREV4D8CXwIuB9ZGxOXd7UqSPj76ut1AG1YCuzPz5wAR8SBwHfBSV7vSR87n/tWPeOf/Tszre6645fHTPuf5ixfyp7ddc9rn1cdXLwTFcuD1lufjwFWtG0TEBmADwCWXXDJ/nekj5eiK3+HcDsw7uHWw5NXTfyT1KAAvnPZ59fHVC0ERJ6jlrCeZ9wL3AgwNDeUJtpdO6oV1nfk/14gT/ROekuk/V535zvg1Cqb2IC5ueT4A7O1SL9KcfVAYGBLqFb0QFM8Bl0XEpRGxCFgDPNblnqQ5ycz/7yb1ijP+0FNmHomIfwQ8AVSALZn5YpfbkqSPjTM+KAAy84+AP+p2H5L0cdQLh54kSV1kUEiSShkUkqRSBoUkqVR81E7Ti4h9wKvd7kP6ABcCb3e7CekE/mJmLjvRCx+5oJDOZBGxIzOHut2HNBceepIklTIoJEmlDAppft3b7QakuXKNQpJUyj0KSVIpg0KSVMqgkD6kiHglIi7sdh9SpxgUUhdFRE98g7M+3gwKaQ4iYklEPB4RfxoROyPit4qX6hHxk4h4ISI+W2y7MiJ+HBE/Le5/uaivj4iHI+K/Aj8q5twSEc8V215XbHdFRDwbET+LiD+LiMu686n1cWdQSHNzLbA3Mz+XmYPAHxf1tzPzC8A9wD8ran8O/Fpmfh74F8C/aZnnV4B1mfnrwAjwVGb+VWAY+LcRsQT4h8DvZeZfAYaY+llgad652yvNzQvAtyNiE/CHmfnfIgLgB8XrzwN/p3h8PnBfsSeQwMKWebZn5v7i8TXAb0bEdMD0A5cATwMjETEA/CAzX+7Uh5LKGBTSHGTm/4yIK4G/CdwRET8qXjpU3E9y7L+r24FmZv7tiFgB/EnLVAdbHgfwdzNz13FvNxYRzwC/ATwREf8gM586fZ9Gao+HnqQ5iIiLgP+Tmb8PfBv4Qsnm5wNvFI/Xl2z3BFNrHFG8x+eL+18Cfp6ZdwGPAX/5w3UvnRqDQpqbvwQ8GxE/Y2pt4V+XbHsnU3sd/x2olGx3O1OHpf4sInYWzwF+C9hZvNdngfs/XOvSqfErPCRJpdyjkCSVMigkSaUMCklSKYNCklTKoJAklTIoJEmlDApJUqn/B1zyUTTRqVZpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.shares.plot.box()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "581be2a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pandas_profiling import ProfileReport\n",
    "\n",
    "# profile = ProfileReport(df.iloc[:, 1:], minimal = True)\n",
    "# profile"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8d38ab6",
   "metadata": {},
   "source": [
    "# Findings\n",
    "\n",
    "* n_unique_tokens, n_non_stop_words, n_non_stop_unique_tokens should be between 0 and 1, because they are proportions +\n",
    "\n",
    "* Check if data_channel_is_* are non-overlaping variables +\n",
    "    - convert them to a categorical variable\n",
    "\n",
    "* \"num_keywords\" indicates how many key words in the article; related variables are kw\\_\\*\\_\\* where * $\\in$ {min,avg,max} +\n",
    "    - When they are -1, it needs to be converted to 0\n",
    "    \n",
    "* \"num_self_hrefs\" indicates how many Marshable's own articles are linked in the article; related variables are self_reference_\\*\\_shares\n",
    "    \n",
    "* weekday_is_\\* and is_weekend can be treated as a single categorical variable\n",
    "\n",
    "* By its proximity to LDA_00, ..., LDA_04, it can be treated as a categotical variable\n",
    "\n",
    "* polarity is between -1 and 1 (negative is between -1 and 0; positive is between 0 and 1)\n",
    "\n",
    "* subjectivity is between 0 and 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e5aa35c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# n_unique_tokens\n",
    "df.n_unique_tokens.lt(0).sum(), df.n_unique_tokens.gt(1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6197e7f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# n_non_stop_words\n",
    "df.n_non_stop_words.lt(0).sum(), df.n_non_stop_words.gt(1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "07f255e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# n_non_stop_words\n",
    "df.n_non_stop_unique_tokens.lt(0).sum(), df.n_non_stop_unique_tokens.gt(1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5e16260d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc[df.n_unique_tokens.le(1),:].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae8581ef",
   "metadata": {},
   "source": [
    "### Data Channel Variables remain as they are, because they need to be represented as in the design matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b979ece0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>timedelta</th>\n",
       "      <th>n_tokens_title</th>\n",
       "      <th>n_tokens_content</th>\n",
       "      <th>n_unique_tokens</th>\n",
       "      <th>n_non_stop_words</th>\n",
       "      <th>n_non_stop_unique_tokens</th>\n",
       "      <th>num_hrefs</th>\n",
       "      <th>num_self_hrefs</th>\n",
       "      <th>num_imgs</th>\n",
       "      <th>...</th>\n",
       "      <th>min_positive_polarity</th>\n",
       "      <th>max_positive_polarity</th>\n",
       "      <th>avg_negative_polarity</th>\n",
       "      <th>min_negative_polarity</th>\n",
       "      <th>max_negative_polarity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "      <th>title_sentiment_polarity</th>\n",
       "      <th>abs_title_subjectivity</th>\n",
       "      <th>abs_title_sentiment_polarity</th>\n",
       "      <th>shares</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://mashable.com/2013/01/07/amazon-instant-...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>0.663594</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.815385</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.350000</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.187500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://mashable.com/2013/01/07/ap-samsung-spon...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>255.0</td>\n",
       "      <td>0.604743</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.791946</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.7</td>\n",
       "      <td>-0.118750</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://mashable.com/2013/01/07/apple-40-billio...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.575130</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.663866</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.466667</td>\n",
       "      <td>-0.800</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://mashable.com/2013/01/07/astronaut-notre...</td>\n",
       "      <td>731.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>531.0</td>\n",
       "      <td>0.503788</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.665635</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.8</td>\n",
       "      <td>-0.369697</td>\n",
       "      <td>-0.600</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://mashable.com/2013/01/07/att-u-verse-apps/</td>\n",
       "      <td>731.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1072.0</td>\n",
       "      <td>0.415646</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540890</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.220192</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  timedelta  \\\n",
       "0  http://mashable.com/2013/01/07/amazon-instant-...      731.0   \n",
       "1  http://mashable.com/2013/01/07/ap-samsung-spon...      731.0   \n",
       "2  http://mashable.com/2013/01/07/apple-40-billio...      731.0   \n",
       "3  http://mashable.com/2013/01/07/astronaut-notre...      731.0   \n",
       "4   http://mashable.com/2013/01/07/att-u-verse-apps/      731.0   \n",
       "\n",
       "   n_tokens_title  n_tokens_content  n_unique_tokens  n_non_stop_words  \\\n",
       "0            12.0             219.0         0.663594               1.0   \n",
       "1             9.0             255.0         0.604743               1.0   \n",
       "2             9.0             211.0         0.575130               1.0   \n",
       "3             9.0             531.0         0.503788               1.0   \n",
       "4            13.0            1072.0         0.415646               1.0   \n",
       "\n",
       "   n_non_stop_unique_tokens  num_hrefs  num_self_hrefs  num_imgs  ...  \\\n",
       "0                  0.815385        4.0             2.0       1.0  ...   \n",
       "1                  0.791946        3.0             1.0       1.0  ...   \n",
       "2                  0.663866        3.0             1.0       1.0  ...   \n",
       "3                  0.665635        9.0             0.0       1.0  ...   \n",
       "4                  0.540890       19.0            19.0      20.0  ...   \n",
       "\n",
       "   min_positive_polarity  max_positive_polarity  avg_negative_polarity  \\\n",
       "0               0.100000                    0.7              -0.350000   \n",
       "1               0.033333                    0.7              -0.118750   \n",
       "2               0.100000                    1.0              -0.466667   \n",
       "3               0.136364                    0.8              -0.369697   \n",
       "4               0.033333                    1.0              -0.220192   \n",
       "\n",
       "   min_negative_polarity  max_negative_polarity  title_subjectivity  \\\n",
       "0                 -0.600              -0.200000            0.500000   \n",
       "1                 -0.125              -0.100000            0.000000   \n",
       "2                 -0.800              -0.133333            0.000000   \n",
       "3                 -0.600              -0.166667            0.000000   \n",
       "4                 -0.500              -0.050000            0.454545   \n",
       "\n",
       "   title_sentiment_polarity  abs_title_subjectivity  \\\n",
       "0                 -0.187500                0.000000   \n",
       "1                  0.000000                0.500000   \n",
       "2                  0.000000                0.500000   \n",
       "3                  0.000000                0.500000   \n",
       "4                  0.136364                0.045455   \n",
       "\n",
       "   abs_title_sentiment_polarity  shares  \n",
       "0                      0.187500     593  \n",
       "1                      0.000000     711  \n",
       "2                      0.000000    1500  \n",
       "3                      0.000000    1200  \n",
       "4                      0.136364     505  \n",
       "\n",
       "[5 rows x 61 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "63b709f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# replacing -1 in kw_*_* to 0\n",
    "df = df.replace({colname: {-1:0} for colname in df.columns[df.columns.str.contains('kw_\\w+_\\w+')]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ce362547",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weekday_is_monday</th>\n",
       "      <th>weekday_is_tuesday</th>\n",
       "      <th>weekday_is_wednesday</th>\n",
       "      <th>weekday_is_thursday</th>\n",
       "      <th>weekday_is_friday</th>\n",
       "      <th>weekday_is_saturday</th>\n",
       "      <th>weekday_is_sunday</th>\n",
       "      <th>is_weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>weekday_is_monday</th>\n",
       "      <td>6661.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_tuesday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>7389.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_wednesday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7435.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_thursday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7267.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_friday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5701.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_saturday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2453.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2453.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weekday_is_sunday</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2737.0</td>\n",
       "      <td>2737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_weekend</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2453.0</td>\n",
       "      <td>2737.0</td>\n",
       "      <td>5190.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      weekday_is_monday  weekday_is_tuesday  \\\n",
       "weekday_is_monday                6661.0                 0.0   \n",
       "weekday_is_tuesday                  0.0              7389.0   \n",
       "weekday_is_wednesday                0.0                 0.0   \n",
       "weekday_is_thursday                 0.0                 0.0   \n",
       "weekday_is_friday                   0.0                 0.0   \n",
       "weekday_is_saturday                 0.0                 0.0   \n",
       "weekday_is_sunday                   0.0                 0.0   \n",
       "is_weekend                          0.0                 0.0   \n",
       "\n",
       "                      weekday_is_wednesday  weekday_is_thursday  \\\n",
       "weekday_is_monday                      0.0                  0.0   \n",
       "weekday_is_tuesday                     0.0                  0.0   \n",
       "weekday_is_wednesday                7435.0                  0.0   \n",
       "weekday_is_thursday                    0.0               7267.0   \n",
       "weekday_is_friday                      0.0                  0.0   \n",
       "weekday_is_saturday                    0.0                  0.0   \n",
       "weekday_is_sunday                      0.0                  0.0   \n",
       "is_weekend                             0.0                  0.0   \n",
       "\n",
       "                      weekday_is_friday  weekday_is_saturday  \\\n",
       "weekday_is_monday                   0.0                  0.0   \n",
       "weekday_is_tuesday                  0.0                  0.0   \n",
       "weekday_is_wednesday                0.0                  0.0   \n",
       "weekday_is_thursday                 0.0                  0.0   \n",
       "weekday_is_friday                5701.0                  0.0   \n",
       "weekday_is_saturday                 0.0               2453.0   \n",
       "weekday_is_sunday                   0.0                  0.0   \n",
       "is_weekend                          0.0               2453.0   \n",
       "\n",
       "                      weekday_is_sunday  is_weekend  \n",
       "weekday_is_monday                   0.0         0.0  \n",
       "weekday_is_tuesday                  0.0         0.0  \n",
       "weekday_is_wednesday                0.0         0.0  \n",
       "weekday_is_thursday                 0.0         0.0  \n",
       "weekday_is_friday                   0.0         0.0  \n",
       "weekday_is_saturday                 0.0      2453.0  \n",
       "weekday_is_sunday                2737.0      2737.0  \n",
       "is_weekend                       2737.0      5190.0  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# No Monday, Thursday, or Friday weekend\n",
    "week = df.loc[:, df.columns.str.contains('week')]\n",
    "week.T.dot(week)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a7b693ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop('is_weekend', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6ae4b5e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We don't need to transform LDA as it indicates the probability each observation falls into "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8d34ae9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['url', 'timedelta', 'n_tokens_title', 'n_tokens_content',\n",
       "       'n_unique_tokens', 'n_non_stop_words', 'n_non_stop_unique_tokens',\n",
       "       'num_hrefs', 'num_self_hrefs', 'num_imgs', 'num_videos',\n",
       "       'average_token_length', 'num_keywords', 'data_channel_is_lifestyle',\n",
       "       'data_channel_is_entertainment', 'data_channel_is_bus',\n",
       "       'data_channel_is_socmed', 'data_channel_is_tech',\n",
       "       'data_channel_is_world', 'kw_min_min', 'kw_max_min', 'kw_avg_min',\n",
       "       'kw_min_max', 'kw_max_max', 'kw_avg_max', 'kw_min_avg', 'kw_max_avg',\n",
       "       'kw_avg_avg', 'self_reference_min_shares', 'self_reference_max_shares',\n",
       "       'self_reference_avg_sharess', 'weekday_is_monday', 'weekday_is_tuesday',\n",
       "       'weekday_is_wednesday', 'weekday_is_thursday', 'weekday_is_friday',\n",
       "       'weekday_is_saturday', 'weekday_is_sunday', 'LDA_00', 'LDA_01',\n",
       "       'LDA_02', 'LDA_03', 'LDA_04', 'global_subjectivity',\n",
       "       'global_sentiment_polarity', 'global_rate_positive_words',\n",
       "       'global_rate_negative_words', 'rate_positive_words',\n",
       "       'rate_negative_words', 'avg_positive_polarity', 'min_positive_polarity',\n",
       "       'max_positive_polarity', 'avg_negative_polarity',\n",
       "       'min_negative_polarity', 'max_negative_polarity', 'title_subjectivity',\n",
       "       'title_sentiment_polarity', 'abs_title_subjectivity',\n",
       "       'abs_title_sentiment_polarity', 'shares'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d9426f2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LDA_00</th>\n",
       "      <th>LDA_01</th>\n",
       "      <th>LDA_02</th>\n",
       "      <th>LDA_03</th>\n",
       "      <th>LDA_04</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LDA_00</th>\n",
       "      <td>4092.480323</td>\n",
       "      <td>622.965962</td>\n",
       "      <td>831.402778</td>\n",
       "      <td>792.584111</td>\n",
       "      <td>978.811497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LDA_01</th>\n",
       "      <td>622.965962</td>\n",
       "      <td>2704.643957</td>\n",
       "      <td>641.946437</td>\n",
       "      <td>940.301129</td>\n",
       "      <td>690.086385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LDA_02</th>\n",
       "      <td>831.402778</td>\n",
       "      <td>641.946437</td>\n",
       "      <td>5010.948823</td>\n",
       "      <td>833.921628</td>\n",
       "      <td>1257.608741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LDA_03</th>\n",
       "      <td>792.584111</td>\n",
       "      <td>940.301129</td>\n",
       "      <td>833.921628</td>\n",
       "      <td>5439.482373</td>\n",
       "      <td>864.833437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LDA_04</th>\n",
       "      <td>978.811497</td>\n",
       "      <td>690.086385</td>\n",
       "      <td>1257.608741</td>\n",
       "      <td>864.833437</td>\n",
       "      <td>5486.520317</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LDA_00       LDA_01       LDA_02       LDA_03       LDA_04\n",
       "LDA_00  4092.480323   622.965962   831.402778   792.584111   978.811497\n",
       "LDA_01   622.965962  2704.643957   641.946437   940.301129   690.086385\n",
       "LDA_02   831.402778   641.946437  5010.948823   833.921628  1257.608741\n",
       "LDA_03   792.584111   940.301129   833.921628  5439.482373   864.833437\n",
       "LDA_04   978.811497   690.086385  1257.608741   864.833437  5486.520317"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = df.loc[:, df.columns.str.contains('LDA_')]\n",
    "lda.T.dot(lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "70c162fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop('n_non_stop_words', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "502f9a73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df['shares_class'] = pd.qcut(df.shares, 20, range(20))\n",
    "df['shares_class'] = df.shares.apply(lambda x: 'pos' if x>1400 else 'neg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "018477b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        neg\n",
       "1        neg\n",
       "2        pos\n",
       "3        neg\n",
       "4        neg\n",
       "        ... \n",
       "39638    pos\n",
       "39639    pos\n",
       "39640    pos\n",
       "39641    neg\n",
       "39642    neg\n",
       "Name: shares_class, Length: 39643, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shares_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "c7fd20a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['url', 'timedelta', 'n_tokens_title', 'n_tokens_content',\n",
       "       'n_unique_tokens', 'n_non_stop_unique_tokens', 'num_hrefs',\n",
       "       'num_self_hrefs', 'num_imgs', 'num_videos', 'average_token_length',\n",
       "       'num_keywords', 'data_channel_is_lifestyle',\n",
       "       'data_channel_is_entertainment', 'data_channel_is_bus',\n",
       "       'data_channel_is_socmed', 'data_channel_is_tech',\n",
       "       'data_channel_is_world', 'kw_min_min', 'kw_max_min', 'kw_avg_min',\n",
       "       'kw_min_max', 'kw_max_max', 'kw_avg_max', 'kw_min_avg', 'kw_max_avg',\n",
       "       'kw_avg_avg', 'self_reference_min_shares', 'self_reference_max_shares',\n",
       "       'self_reference_avg_sharess', 'weekday_is_monday', 'weekday_is_tuesday',\n",
       "       'weekday_is_wednesday', 'weekday_is_thursday', 'weekday_is_friday',\n",
       "       'weekday_is_saturday', 'weekday_is_sunday', 'LDA_00', 'LDA_01',\n",
       "       'LDA_02', 'LDA_03', 'LDA_04', 'global_subjectivity',\n",
       "       'global_sentiment_polarity', 'global_rate_positive_words',\n",
       "       'global_rate_negative_words', 'rate_positive_words',\n",
       "       'rate_negative_words', 'avg_positive_polarity', 'min_positive_polarity',\n",
       "       'max_positive_polarity', 'avg_negative_polarity',\n",
       "       'min_negative_polarity', 'max_negative_polarity', 'title_subjectivity',\n",
       "       'title_sentiment_polarity', 'abs_title_subjectivity',\n",
       "       'abs_title_sentiment_polarity', 'shares', 'shares_class'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "de9cebd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(['url','timedelta','shares'], axis = 1).to_csv('text_summary_stats.csv', \n",
    "                                                             index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3dd758b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# run the python script that obtain texts from url\n",
    "!python data_prep.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}