naive_bayes_spam_user.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "devoted-postage",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlflow\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import fbeta_score, accuracy_score, precision_score, recall_score\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "champion-allergy",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data_path =\"data/*.csv\"\n",
    "holdout_data_path = \"data/holdout/*.csv\"\n",
    "train_on_holdout = False\n",
    "input_columns = [\"first_name\", \"last_name\", \"email\"]\n",
    "test_size = 0.2\n",
    "experiment_id = 3421272196413697\n",
    "random_state = 42\n",
    "betas = [1, 1/2, 1/8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "spare-friendship",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow.set_tracking_uri(\"databricks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "innovative-welsh",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow.sklearn.autolog(log_input_examples=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "satisfactory-intermediate",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(path, input_cols):\n",
    "    dfs = []\n",
    "    for fpath in glob.glob(path):\n",
    "        temp_df = pd.read_csv(fpath).fillna(\"\")[input_cols].drop_duplicates()\n",
    "        temp_df[\"source\"] = fpath.split(\"/\")[-1].split(\".\")[0]\n",
    "        if \"real\" in fpath:\n",
    "            temp_df[\"is_fake\"] = 0\n",
    "        else:\n",
    "            temp_df[\"is_fake\"] = 1\n",
    "        dfs.append(temp_df)\n",
    "    df = pd.concat(dfs)\n",
    "    del temp_df\n",
    "    del dfs\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "proud-input",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_df = read_data(input_data_path, input_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "challenging-spotlight",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>is_fake</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>innapropiate_spam_fake_users</th>\n",
       "      <td>0</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>real_fake_users</th>\n",
       "      <td>500000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam_fake_users</th>\n",
       "      <td>0</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "is_fake                            0       1\n",
       "source                                      \n",
       "innapropiate_spam_fake_users       0  250000\n",
       "real_fake_users               500000       0\n",
       "spam_fake_users                    0  250000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(input_df[\"source\"], input_df[\"is_fake\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "overall-psychology",
   "metadata": {},
   "outputs": [],
   "source": [
    "holdout_df = read_data(holdout_data_path, input_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "blocked-cooler",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>is_fake</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>generated_real_like</th>\n",
       "      <td>270</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>manual_labeled_data_ch</th>\n",
       "      <td>0</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "is_fake                   0    1\n",
       "source                          \n",
       "generated_real_like     270    0\n",
       "manual_labeled_data_ch    0  270"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(holdout_df[\"source\"], holdout_df[\"is_fake\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "solar-banana",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing traing and test set...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021/03/11 15:28:23 WARNING mlflow.sklearn.utils: Truncated the value of the key `steps`. Truncated value: `[('prep', ColumnTransformer(remainder='passthrough',\n",
      "                  transformers=[('first_name',\n",
      "                                 TfidfVectorizer(analyzer='char',\n",
      "                                                 lowercase=False,\n",
      "               ...`\n",
      "2021/03/11 15:28:23 WARNING mlflow.sklearn.utils: Truncated the value of the key `prep`. Truncated value: `ColumnTransformer(remainder='passthrough',\n",
      "                  transformers=[('first_name',\n",
      "                                 TfidfVectorizer(analyzer='char',\n",
      "                                                 lowercase=False,\n",
      "                         ...`\n",
      "2021/03/11 15:28:23 WARNING mlflow.sklearn.utils: Truncated the value of the key `prep__transformers`. Truncated value: `[('first_name', TfidfVectorizer(analyzer='char', lowercase=False, max_features=50, min_df=0.1,\n",
      "                ngram_range=(1, 2)), 'first_name'), ('last_name', TfidfVectorizer(analyzer='char', lowercase=False, max_features=50, min_df=0.1,\n",
      "       ...`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting pipeline...\n",
      "Evaluating pipeline...\n",
      "Finished.\n"
     ]
    }
   ],
   "source": [
    "with mlflow.start_run(experiment_id=experiment_id) as run:\n",
    "    if train_on_holdout:\n",
    "        del input_df\n",
    "        input_df = holdout_df\n",
    "    mlflow.log_param(\"train_on_holdout\", train_on_holdout)\n",
    "    \n",
    "    print(\"Preparing traing and test set...\")\n",
    "    y = input_df.is_fake\n",
    "    X = input_df.drop(['is_fake'], axis=1)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state, stratify=y)\n",
    "    \n",
    "    train_source = X_train.source\n",
    "    X_train = X_train.drop(['source'], axis=1)\n",
    "    \n",
    "    test_source = X_test.source\n",
    "    X_test = X_test.drop(['source'], axis=1)\n",
    "    \n",
    "    mlflow.log_param(\"input_data_path\", input_data_path)\n",
    "    mlflow.log_param(\"holdout_data_path\", holdout_data_path)\n",
    "    mlflow.log_param(\"test_size\", test_size)\n",
    "    \n",
    "    vectorizer_params = {\n",
    "        \"encoding\":\"utf-8\",\n",
    "        \"strip_accents\":None,\n",
    "        \"lowercase\":False,\n",
    "        \"analyzer\":\"char\",\n",
    "        \"stop_words\":None,\n",
    "        \"max_df\":1.0,\n",
    "        \"min_df\":0.1,\n",
    "        \"max_features\":50,\n",
    "        \"use_idf\":True\n",
    "    }\n",
    "    \n",
    "    ct = ColumnTransformer([(\"first_name\", TfidfVectorizer(**vectorizer_params, ngram_range=(1,2)), \"first_name\"),\n",
    "                            (\"last_name\", TfidfVectorizer(**vectorizer_params, ngram_range=(1,2)), \"last_name\"),\n",
    "                            (\"email\", TfidfVectorizer(**vectorizer_params, ngram_range=(1,3)), \"email\")\n",
    "                           ],\n",
    "                         remainder=\"passthrough\")\n",
    "    print(\"Fitting pipeline...\")\n",
    "    pipe = Pipeline([('prep', ct), ('clf', MultinomialNB())])\n",
    "        \n",
    "    pipe.fit(X_train, y_train)\n",
    "    \n",
    "    print(\"Evaluating pipeline...\")\n",
    "    y_pred = pipe.predict(X_test)\n",
    "    \n",
    "    mlflow.log_metric(\"accuracy_test\", accuracy_score(y_test, y_pred))\n",
    "    mlflow.log_metric(\"precision_test\", precision_score(y_test, y_pred))\n",
    "    mlflow.log_metric(\"recall_test\", recall_score(y_test, y_pred))\n",
    "    \n",
    "    if not train_on_holdout:\n",
    "        holdout_y_pred = pipe.predict(holdout_df.drop([\"is_fake\", \"source\"], axis=1))\n",
    "        mlflow.log_metric(\"accuracy_holdout\", accuracy_score(holdout_df.is_fake, holdout_y_pred))\n",
    "        mlflow.log_metric(\"precision_holdout\", precision_score(holdout_df.is_fake, holdout_y_pred))\n",
    "        mlflow.log_metric(\"recall_holdout\", recall_score(holdout_df.is_fake, holdout_y_pred))\n",
    "    \n",
    "    \n",
    "    for b in betas:\n",
    "        mlflow.log_metric(\"f_beta_test_{}\".format(b), fbeta_score(y_test, y_pred, beta=b))\n",
    "        if not train_on_holdout:\n",
    "            mlflow.log_metric(\"f_beta_holdout_{}\".format(b), fbeta_score(holdout_df.is_fake, holdout_y_pred, beta=b))\n",
    "    \n",
    "    \n",
    "    # mlflow.shap.log_explanation(pipe.predict, X_train)\n",
    "    X_test.loc[:,\"is_fake\"] = y_test\n",
    "    X_test.loc[:,\"is_fake_pred\"] = y_pred\n",
    "    X_test.loc[:, \"source\"] = test_source\n",
    "    X_test.to_csv(\"test_data.csv\", index=False)\n",
    "    mlflow.log_artifact(\"test_data.csv\")\n",
    "    \n",
    "    if not train_on_holdout:\n",
    "        holdout_df.loc[:, \"is_fake_pred\"] = holdout_y_pred\n",
    "        holdout_df.to_csv(\"holdout_data.csv\", index=False)\n",
    "        mlflow.log_artifact(\"holdout_data.csv\")\n",
    "    print(\"Finished.\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "senior-fitness",
   "metadata": {},
   "outputs": [],
   "source": [
    "if train_on_holdout:\n",
    "    demo_df = X_test\n",
    "else:\n",
    "    demo_df = holdout_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "antique-painting",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Improvements\n",
    "# - log more metrics\n",
    "# - start and end characters for first name and last name\n",
    "# - play with tfidf parameters\n",
    "# - add shap or feature probabilities\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}