classifier_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:26.355129Z",
     "start_time": "2020-09-15T00:59:26.262217Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext nb_black\n",
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:26.860713Z",
     "start_time": "2020-09-15T00:59:26.356211Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "\n",
    "import os\n",
    "from requests import get\n",
    "from pathlib import Path\n",
    "import gc\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from zipfile import ZipFile\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:27.766098Z",
     "start_time": "2020-09-15T00:59:26.861936Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:27.891616Z",
     "start_time": "2020-09-15T00:59:27.767821Z"
    }
   },
   "outputs": [],
   "source": [
    "from thc_net.classifier import ThcNetClassifier\n",
    "from thc_net.utils import download, plot_history\n",
    "from thc_net.input_utils import prepare_input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bank marketing  : loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.004274Z",
     "start_time": "2020-09-15T00:59:27.892699Z"
    }
   },
   "outputs": [],
   "source": [
    "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank-additional.zip\"\n",
    "dataset_name = \"bank-additional-full\"\n",
    "out_zip = Path(os.getcwd().rsplit(\"/\", 1)[0] + \"/data/bank-additional.zip\")\n",
    "out = Path(\n",
    "    os.getcwd().rsplit(\"/\", 1)[0] + \"/data/bank-additional/\" + dataset_name + \".csv\"\n",
    ")\n",
    "\n",
    "download(url, out_zip)\n",
    "with ZipFile(out_zip, \"r\") as zipObj:\n",
    "    zipObj.extractall(\"/data\")\n",
    "\n",
    "target = \"y\"\n",
    "to_remove = []\n",
    "train = pd.read_csv(out, sep=\";\", low_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Census income  : loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.071907Z",
     "start_time": "2020-09-15T00:59:28.005422Z"
    }
   },
   "outputs": [],
   "source": [
    "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\"\n",
    "dataset_name = \"census-income\"\n",
    "out = Path(os.getcwd().rsplit(\"/\", 1)[0] + \"/data/\" + dataset_name + \".csv\")\n",
    "\n",
    "\n",
    "download(url, out)\n",
    "target = \" <=50K\"\n",
    "to_remove = []\n",
    "train = pd.read_csv(out, sep=\",\", low_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data and split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.119480Z",
     "start_time": "2020-09-15T00:59:28.073089Z"
    }
   },
   "outputs": [],
   "source": [
    "if \"Set\" not in train.columns:\n",
    "    train[\"Set\"] = np.random.choice(\n",
    "        [\"train\", \"valid\", \"test\"], p=[0.8, 0.1, 0.1], size=(train.shape[0],)\n",
    "    )\n",
    "\n",
    "train_indices = train[train.Set == \"train\"].index\n",
    "valid_indices = train[train.Set == \"valid\"].index\n",
    "test_indices = train[train.Set == \"test\"].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.147175Z",
     "start_time": "2020-09-15T00:59:28.121111Z"
    }
   },
   "outputs": [],
   "source": [
    "Y = train[target].values\n",
    "X = train.drop(columns=[\"Set\"] + [target])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.171365Z",
     "start_time": "2020-09-15T00:59:28.148556Z"
    }
   },
   "outputs": [],
   "source": [
    "ratio = 0.005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.231213Z",
     "start_time": "2020-09-15T00:59:28.172466Z"
    }
   },
   "outputs": [],
   "source": [
    "n_unique = X.nunique()\n",
    "ratios = (n_unique / X.shape[0]) < ratio\n",
    "cat_idxs = np.argwhere(\n",
    "    X.columns.isin(X.columns[ratios | (X.dtypes == \"object\")])\n",
    ").ravel()\n",
    "cat_dims = n_unique[cat_idxs].values + X.isnull().sum()[cat_idxs].values + 1\n",
    "del n_unique, ratios, train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.304280Z",
     "start_time": "2020-09-15T00:59:28.232178Z"
    }
   },
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.334367Z",
     "start_time": "2020-09-15T00:59:28.305189Z"
    }
   },
   "outputs": [],
   "source": [
    "tg_enc = LabelEncoder()\n",
    "Y = tg_enc.fit_transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.382875Z",
     "start_time": "2020-09-15T00:59:28.335440Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train = X.values[train_indices]\n",
    "y_train = Y[train_indices]\n",
    "\n",
    "X_valid = X.values[valid_indices]\n",
    "y_valid = Y[valid_indices]\n",
    "\n",
    "X_test = X.values[test_indices]\n",
    "y_test = Y[test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.407041Z",
     "start_time": "2020-09-15T00:59:28.383991Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.708027Z",
     "start_time": "2020-09-15T00:59:28.408031Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train_prep, encoders = prepare_input_data(X_train, cat_idxs)\n",
    "X_valid_prep, _ = prepare_input_data(X_valid, cat_idxs, encoders=encoders, fit=False)\n",
    "X_test_prep, _ = prepare_input_data(X_test, cat_idxs, encoders=encoders, fit=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:28.731489Z",
     "start_time": "2020-09-15T00:59:28.708977Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics = [\"AUC\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:59:57.058405Z",
     "start_time": "2020-09-15T00:59:28.732715Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "model = ThcNetClassifier(\n",
    "    n_layer=3,\n",
    "    mul_input=8, \n",
    "    metrics=metrics,\n",
    "    cat_idxs=cat_idxs,\n",
    "    cat_emb_dims=cat_dims,\n",
    "    dropout=0.05,\n",
    "    normalize=False,\n",
    "    max_emb=20,\n",
    "    patience=30\n",
    ")\n",
    "\n",
    "history = model.fit(\n",
    "        X=X_train_prep, \n",
    "        y=y_train,\n",
    "        X_valid=X_valid_prep,\n",
    "        y_valid=y_valid,\n",
    "        batch_size=1024,\n",
    "        epochs=10000,\n",
    "        verbose=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.155919Z",
     "start_time": "2020-09-15T01:02:36.071253Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "model_snn = ThcNetClassifier(\n",
    "    n_layer=3,\n",
    "    mul_input=8, \n",
    "    metrics=metrics,\n",
    "    cat_idxs=cat_idxs,\n",
    "    cat_emb_dims=cat_dims,\n",
    "    dropout=0.05,\n",
    "    normalize=False,\n",
    "    max_emb=20,\n",
    "    patience=30,\n",
    "    use_snn=True,\n",
    "    noise=None\n",
    ")\n",
    "\n",
    "history_snn = model_snn.fit(\n",
    "        X=X_train_prep, \n",
    "        y=y_train,\n",
    "        X_valid=X_valid_prep,\n",
    "        y_valid=y_valid,\n",
    "        batch_size=1024,\n",
    "        epochs=10000,\n",
    "        verbose=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.187579Z",
     "start_time": "2020-09-15T01:03:36.157073Z"
    }
   },
   "outputs": [],
   "source": [
    "model.network.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.215380Z",
     "start_time": "2020-09-15T01:03:36.188617Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_model(\n",
    "    model,\n",
    "    # to_file=\"model.png\",\n",
    "    show_shapes=True,\n",
    "    show_layer_names=True,\n",
    "    rankdir=\"TB\",\n",
    "    expand_nested=False,\n",
    "    dpi=96,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.246732Z",
     "start_time": "2020-09-15T01:03:36.216328Z"
    }
   },
   "outputs": [],
   "source": [
    "model_snn.network.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.410872Z",
     "start_time": "2020-09-15T01:03:36.247630Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.562386Z",
     "start_time": "2020-09-15T01:03:36.412073Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_history(history_snn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:36.727379Z",
     "start_time": "2020-09-15T01:03:36.563270Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict_proba(X_test_prep)\n",
    "test_auc = roc_auc_score(y_score=y_pred[:, 1], y_true=y_test)\n",
    "print(f\"FINAL TEST SCORE FOR {dataset_name} : {test_auc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T01:03:37.018598Z",
     "start_time": "2020-09-15T01:03:36.728866Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = model_snn.predict_proba(X_test_prep)\n",
    "test_auc = roc_auc_score(y_score=y_pred[:, 1], y_true=y_test)\n",
    "print(f\"FINAL TEST SCORE FOR {dataset_name} : {test_auc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}