predict.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>x8</th>\n",
       "      <th>x9</th>\n",
       "      <th>...</th>\n",
       "      <th>x91</th>\n",
       "      <th>x92</th>\n",
       "      <th>x93</th>\n",
       "      <th>x94</th>\n",
       "      <th>x95</th>\n",
       "      <th>x96</th>\n",
       "      <th>x97</th>\n",
       "      <th>x98</th>\n",
       "      <th>x99</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.247824</td>\n",
       "      <td>6.425853</td>\n",
       "      <td>-6.485658</td>\n",
       "      <td>-8.241462</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.03%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-9.808905</td>\n",
       "      <td>4.753455</td>\n",
       "      <td>2.537819</td>\n",
       "      <td>...</td>\n",
       "      <td>0.592543</td>\n",
       "      <td>14.967527</td>\n",
       "      <td>6.691044</td>\n",
       "      <td>-0.275572</td>\n",
       "      <td>2.518542</td>\n",
       "      <td>4.637431</td>\n",
       "      <td>-14.296510</td>\n",
       "      <td>1.898115</td>\n",
       "      <td>-1.225228</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.623274</td>\n",
       "      <td>-2.808263</td>\n",
       "      <td>5.321746</td>\n",
       "      <td>-2.437653</td>\n",
       "      <td>0.614041</td>\n",
       "      <td>0.02%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>10.710053</td>\n",
       "      <td>1.608882</td>\n",
       "      <td>-2.027541</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.712788</td>\n",
       "      <td>-3.638637</td>\n",
       "      <td>-2.256492</td>\n",
       "      <td>2.703101</td>\n",
       "      <td>1.760729</td>\n",
       "      <td>2.660614</td>\n",
       "      <td>1.182922</td>\n",
       "      <td>-3.203078</td>\n",
       "      <td>0.711018</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-5.460551</td>\n",
       "      <td>5.997701</td>\n",
       "      <td>-4.614194</td>\n",
       "      <td>-6.035128</td>\n",
       "      <td>-1.986438</td>\n",
       "      <td>-0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-3.364649</td>\n",
       "      <td>-2.493473</td>\n",
       "      <td>1.429913</td>\n",
       "      <td>...</td>\n",
       "      <td>2.206956</td>\n",
       "      <td>7.292558</td>\n",
       "      <td>10.095556</td>\n",
       "      <td>1.601155</td>\n",
       "      <td>1.710645</td>\n",
       "      <td>-12.519434</td>\n",
       "      <td>-5.261707</td>\n",
       "      <td>2.032666</td>\n",
       "      <td>-6.445528</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.566307</td>\n",
       "      <td>-0.080517</td>\n",
       "      <td>0.064754</td>\n",
       "      <td>6.063193</td>\n",
       "      <td>-2.253075</td>\n",
       "      <td>0.0%</td>\n",
       "      <td>Thurday</td>\n",
       "      <td>-6.111982</td>\n",
       "      <td>-1.796377</td>\n",
       "      <td>-3.169510</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.841669</td>\n",
       "      <td>5.189840</td>\n",
       "      <td>10.291069</td>\n",
       "      <td>-3.642396</td>\n",
       "      <td>0.674300</td>\n",
       "      <td>-1.104317</td>\n",
       "      <td>-3.922342</td>\n",
       "      <td>-0.613947</td>\n",
       "      <td>7.275179</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.750715</td>\n",
       "      <td>15.656095</td>\n",
       "      <td>-6.778600</td>\n",
       "      <td>11.251037</td>\n",
       "      <td>-0.308200</td>\n",
       "      <td>0.0%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>2.498892</td>\n",
       "      <td>-0.323229</td>\n",
       "      <td>-2.624680</td>\n",
       "      <td>...</td>\n",
       "      <td>8.933781</td>\n",
       "      <td>-7.295794</td>\n",
       "      <td>-11.246059</td>\n",
       "      <td>-4.876594</td>\n",
       "      <td>3.395415</td>\n",
       "      <td>-2.646844</td>\n",
       "      <td>-5.273267</td>\n",
       "      <td>-5.259536</td>\n",
       "      <td>-3.240523</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>-2.664967</td>\n",
       "      <td>-3.427965</td>\n",
       "      <td>-5.143886</td>\n",
       "      <td>0.289251</td>\n",
       "      <td>-3.202695</td>\n",
       "      <td>0.01%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-13.356218</td>\n",
       "      <td>0.244953</td>\n",
       "      <td>-2.524016</td>\n",
       "      <td>...</td>\n",
       "      <td>2.123462</td>\n",
       "      <td>6.139848</td>\n",
       "      <td>-4.323045</td>\n",
       "      <td>-6.453976</td>\n",
       "      <td>6.345870</td>\n",
       "      <td>6.482477</td>\n",
       "      <td>-11.562510</td>\n",
       "      <td>4.216840</td>\n",
       "      <td>5.689256</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>2.174302</td>\n",
       "      <td>-4.912113</td>\n",
       "      <td>-4.803975</td>\n",
       "      <td>-22.081911</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-0.266593</td>\n",
       "      <td>-1.743051</td>\n",
       "      <td>5.870177</td>\n",
       "      <td>...</td>\n",
       "      <td>-12.307131</td>\n",
       "      <td>-4.762671</td>\n",
       "      <td>10.574798</td>\n",
       "      <td>-3.954958</td>\n",
       "      <td>0.998331</td>\n",
       "      <td>9.001036</td>\n",
       "      <td>1.171144</td>\n",
       "      <td>-1.001999</td>\n",
       "      <td>9.383899</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>2.247079</td>\n",
       "      <td>-3.377818</td>\n",
       "      <td>-1.567924</td>\n",
       "      <td>-7.445187</td>\n",
       "      <td>-0.677291</td>\n",
       "      <td>-0.01%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>6.618001</td>\n",
       "      <td>2.323299</td>\n",
       "      <td>3.724704</td>\n",
       "      <td>...</td>\n",
       "      <td>-8.084817</td>\n",
       "      <td>8.031024</td>\n",
       "      <td>7.282587</td>\n",
       "      <td>2.439789</td>\n",
       "      <td>1.666460</td>\n",
       "      <td>0.566402</td>\n",
       "      <td>3.591239</td>\n",
       "      <td>2.962485</td>\n",
       "      <td>0.980093</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>-1.402385</td>\n",
       "      <td>-20.117766</td>\n",
       "      <td>3.705099</td>\n",
       "      <td>11.634801</td>\n",
       "      <td>0.680733</td>\n",
       "      <td>-0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-4.304220</td>\n",
       "      <td>-1.730289</td>\n",
       "      <td>-12.788627</td>\n",
       "      <td>...</td>\n",
       "      <td>4.920424</td>\n",
       "      <td>-0.954106</td>\n",
       "      <td>6.261179</td>\n",
       "      <td>0.628781</td>\n",
       "      <td>-0.109908</td>\n",
       "      <td>2.086040</td>\n",
       "      <td>12.889705</td>\n",
       "      <td>7.359320</td>\n",
       "      <td>0.124861</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>-5.761582</td>\n",
       "      <td>8.638806</td>\n",
       "      <td>8.391700</td>\n",
       "      <td>-6.474205</td>\n",
       "      <td>-1.642141</td>\n",
       "      <td>-0.02%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>-14.284262</td>\n",
       "      <td>1.382704</td>\n",
       "      <td>3.687798</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.721215</td>\n",
       "      <td>5.300117</td>\n",
       "      <td>-13.373868</td>\n",
       "      <td>1.450768</td>\n",
       "      <td>1.498209</td>\n",
       "      <td>6.701758</td>\n",
       "      <td>-2.233975</td>\n",
       "      <td>12.776734</td>\n",
       "      <td>0.565346</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            x0         x1        x2         x3        x4      x5         x6  \\\n",
       "0    -0.247824   6.425853 -6.485658  -8.241462       NaN   0.03%    Tuesday   \n",
       "1     2.623274  -2.808263  5.321746  -2.437653  0.614041   0.02%  Wednesday   \n",
       "2    -5.460551   5.997701 -4.614194  -6.035128 -1.986438   -0.0%    Tuesday   \n",
       "3     1.566307  -0.080517  0.064754   6.063193 -2.253075    0.0%    Thurday   \n",
       "4     0.750715  15.656095 -6.778600  11.251037 -0.308200    0.0%  Wednesday   \n",
       "...        ...        ...       ...        ...       ...     ...        ...   \n",
       "7995 -2.664967  -3.427965 -5.143886   0.289251 -3.202695   0.01%    Tuesday   \n",
       "7996  2.174302  -4.912113 -4.803975 -22.081911       NaN    0.0%    Tuesday   \n",
       "7997  2.247079  -3.377818 -1.567924  -7.445187 -0.677291  -0.01%  Wednesday   \n",
       "7998 -1.402385 -20.117766  3.705099  11.634801  0.680733   -0.0%    Tuesday   \n",
       "7999 -5.761582   8.638806  8.391700  -6.474205 -1.642141  -0.02%    Tuesday   \n",
       "\n",
       "             x7        x8         x9  ...        x91        x92        x93  \\\n",
       "0     -9.808905  4.753455   2.537819  ...   0.592543  14.967527   6.691044   \n",
       "1     10.710053  1.608882  -2.027541  ...  -1.712788  -3.638637  -2.256492   \n",
       "2     -3.364649 -2.493473   1.429913  ...   2.206956   7.292558  10.095556   \n",
       "3     -6.111982 -1.796377  -3.169510  ...  -3.841669   5.189840  10.291069   \n",
       "4      2.498892 -0.323229  -2.624680  ...   8.933781  -7.295794 -11.246059   \n",
       "...         ...       ...        ...  ...        ...        ...        ...   \n",
       "7995 -13.356218  0.244953  -2.524016  ...   2.123462   6.139848  -4.323045   \n",
       "7996  -0.266593 -1.743051   5.870177  ... -12.307131  -4.762671  10.574798   \n",
       "7997   6.618001  2.323299   3.724704  ...  -8.084817   8.031024   7.282587   \n",
       "7998  -4.304220 -1.730289 -12.788627  ...   4.920424  -0.954106   6.261179   \n",
       "7999 -14.284262  1.382704   3.687798  ...  -4.721215   5.300117 -13.373868   \n",
       "\n",
       "           x94       x95        x96        x97        x98       x99  y  \n",
       "0    -0.275572  2.518542   4.637431 -14.296510   1.898115 -1.225228  0  \n",
       "1     2.703101  1.760729   2.660614   1.182922  -3.203078  0.711018  0  \n",
       "2     1.601155  1.710645 -12.519434  -5.261707   2.032666 -6.445528  0  \n",
       "3    -3.642396  0.674300  -1.104317  -3.922342  -0.613947  7.275179  0  \n",
       "4    -4.876594  3.395415  -2.646844  -5.273267  -5.259536 -3.240523  0  \n",
       "...        ...       ...        ...        ...        ...       ... ..  \n",
       "7995 -6.453976  6.345870   6.482477 -11.562510   4.216840  5.689256  0  \n",
       "7996 -3.954958  0.998331   9.001036   1.171144  -1.001999  9.383899  0  \n",
       "7997  2.439789  1.666460   0.566402   3.591239   2.962485  0.980093  0  \n",
       "7998  0.628781 -0.109908   2.086040  12.889705   7.359320  0.124861  0  \n",
       "7999  1.450768  1.498209   6.701758  -2.233975  12.776734  0.565346  0  \n",
       "\n",
       "[8000 rows x 101 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.pipeline import Pipeline as ImbPipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "historical_df = pd.read_csv('./historical_transaction.csv')\n",
    "current_df = pd.read_csv('./current_transaction.csv')\n",
    "historical_df_without_y = historical_df.drop(columns='y')\n",
    "\n",
    "historical_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float64    93\n",
       "object      6\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "historical_df_without_y.dtypes[:-1].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our dataset, there are six columns identified as categorical data types (`object`), which need to be converted into integer values before they can be effectively utilized in our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x20</th>\n",
       "      <th>x27</th>\n",
       "      <th>x49</th>\n",
       "      <th>x57</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.03%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Aug</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>False</td>\n",
       "      <td>$1578.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Jul</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>False</td>\n",
       "      <td>$767.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Aug</td>\n",
       "      <td>Evening</td>\n",
       "      <td>False</td>\n",
       "      <td>$931.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>Thurday</td>\n",
       "      <td>Aug</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>True</td>\n",
       "      <td>$2124.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Jul</td>\n",
       "      <td>Evening</td>\n",
       "      <td>False</td>\n",
       "      <td>$-3055.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>0.01%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Jun</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>False</td>\n",
       "      <td>$-1664.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>May</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>False</td>\n",
       "      <td>$-42.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>-0.01%</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Aug</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>True</td>\n",
       "      <td>$220.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>-0.0%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Aug</td>\n",
       "      <td>Evening</td>\n",
       "      <td>True</td>\n",
       "      <td>$467.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>-0.02%</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Jun</td>\n",
       "      <td>Afternoon</td>\n",
       "      <td>False</td>\n",
       "      <td>$-598.35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          x5         x6  x20        x27    x49        x57\n",
       "0      0.03%    Tuesday  Aug  Afternoon  False   $1578.34\n",
       "1      0.02%  Wednesday  Jul  Afternoon  False    $767.76\n",
       "2      -0.0%    Tuesday  Aug    Evening  False    $931.06\n",
       "3       0.0%    Thurday  Aug  Afternoon   True   $2124.52\n",
       "4       0.0%  Wednesday  Jul    Evening  False  $-3055.67\n",
       "...      ...        ...  ...        ...    ...        ...\n",
       "7995   0.01%    Tuesday  Jun  Afternoon  False  $-1664.04\n",
       "7996    0.0%    Tuesday  May  Afternoon  False    $-42.81\n",
       "7997  -0.01%  Wednesday  Aug  Afternoon   True    $220.59\n",
       "7998   -0.0%    Tuesday  Aug    Evening   True    $467.47\n",
       "7999  -0.02%    Tuesday  Jun  Afternoon  False   $-598.35\n",
       "\n",
       "[8000 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "historical_df_without_y.select_dtypes(include='object')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Column `x5`**:\n",
    "  - The values in this column represent percentages stored as strings with a trailing '%'. \n",
    "  - To use these values for numerical analysis, we first remove the '%' sign from each string, and then convert the resulting strings to floating-point numbers.\n",
    "\n",
    "- **Column `x57`**:\n",
    "  - This column contains monetary values represented as strings prefixed with a dollar sign ('$').\n",
    "  - Similar to the transformation for column `x5`, we remove the '$' sign and convert the strings to floats."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x5</th>\n",
       "      <th>x57</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>1578.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0002</td>\n",
       "      <td>767.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.0000</td>\n",
       "      <td>931.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>2124.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>-3055.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>-1664.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>-42.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>-0.0001</td>\n",
       "      <td>220.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>-0.0000</td>\n",
       "      <td>467.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>-0.0002</td>\n",
       "      <td>-598.35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          x5      x57\n",
       "0     0.0003  1578.34\n",
       "1     0.0002   767.76\n",
       "2    -0.0000   931.06\n",
       "3     0.0000  2124.52\n",
       "4     0.0000 -3055.67\n",
       "...      ...      ...\n",
       "7995  0.0001 -1664.04\n",
       "7996  0.0000   -42.81\n",
       "7997 -0.0001   220.59\n",
       "7998 -0.0000   467.47\n",
       "7999 -0.0002  -598.35\n",
       "\n",
       "[8000 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "historical_df_without_y['x5'] = historical_df_without_y['x5'].str.rstrip('%').astype('float') / 100.0\n",
    "historical_df_without_y['x57'] = historical_df_without_y['x57'].astype(str).replace('[\\$,]', '', regex=True).astype('float')\n",
    "\n",
    "historical_df_without_y[['x5', 'x57']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To prepare the dataset for model training, we undertake a series of preprocessing steps.\n",
    "\n",
    "- **Numerical Columns**:\n",
    "  - We attempt to select columns of type `int64`, assuming these represent numerical data.\n",
    "\n",
    "- **Categorical Columns**:\n",
    "  - We explicitly define a list of columns that are known to contain categorical data: `['x6', 'x20', 'x27', 'x49']`.\n",
    "\n",
    "- **Numerical Pipeline**:\n",
    "  - **Imputer**: Missing values in numerical columns are replaced with the median of each column.\n",
    "  - **Scaler**: StandardScaler is applied to normalize the data, ensuring that each feature contributes equally to model training by bringing them to a common scale.\n",
    "\n",
    "- **Categorical Pipeline**:\n",
    "  - **Imputer**: For categorical columns, missing values are replaced with the most frequent value in each column.\n",
    "  - **One-Hot Encoder**: Converts categorical variables into a form that could be provided to ML algorithms. This encoder creates binary columns for each category/level and handles unknown categories by ignoring them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x10</th>\n",
       "      <th>x11</th>\n",
       "      <th>x12</th>\n",
       "      <th>x13</th>\n",
       "      <th>x14</th>\n",
       "      <th>x15</th>\n",
       "      <th>x16</th>\n",
       "      <th>x17</th>\n",
       "      <th>...</th>\n",
       "      <th>x20_May</th>\n",
       "      <th>x20_Nov</th>\n",
       "      <th>x20_Oct</th>\n",
       "      <th>x20_Sept</th>\n",
       "      <th>x27_Afternoon</th>\n",
       "      <th>x27_Evening</th>\n",
       "      <th>x27_Morning</th>\n",
       "      <th>x27_Night</th>\n",
       "      <th>x49_False</th>\n",
       "      <th>x49_True</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.075227</td>\n",
       "      <td>0.431298</td>\n",
       "      <td>-0.102254</td>\n",
       "      <td>1.120197</td>\n",
       "      <td>0.260087</td>\n",
       "      <td>-1.199619</td>\n",
       "      <td>-2.450272</td>\n",
       "      <td>-1.659281</td>\n",
       "      <td>1.102589</td>\n",
       "      <td>2.012662</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.626326</td>\n",
       "      <td>-0.229215</td>\n",
       "      <td>0.293587</td>\n",
       "      <td>0.199684</td>\n",
       "      <td>0.331408</td>\n",
       "      <td>0.632379</td>\n",
       "      <td>-0.225701</td>\n",
       "      <td>-0.157557</td>\n",
       "      <td>-0.516379</td>\n",
       "      <td>-0.119575</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.348959</td>\n",
       "      <td>0.400672</td>\n",
       "      <td>-0.934314</td>\n",
       "      <td>-1.947026</td>\n",
       "      <td>-0.016606</td>\n",
       "      <td>0.222856</td>\n",
       "      <td>-1.347693</td>\n",
       "      <td>-0.992515</td>\n",
       "      <td>-1.378931</td>\n",
       "      <td>-2.035756</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.368056</td>\n",
       "      <td>-0.034100</td>\n",
       "      <td>-0.384268</td>\n",
       "      <td>0.119804</td>\n",
       "      <td>-0.373455</td>\n",
       "      <td>-1.815058</td>\n",
       "      <td>-0.902405</td>\n",
       "      <td>-0.288641</td>\n",
       "      <td>-0.613191</td>\n",
       "      <td>-1.453367</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.168766</td>\n",
       "      <td>1.091534</td>\n",
       "      <td>2.082734</td>\n",
       "      <td>-0.515513</td>\n",
       "      <td>2.155874</td>\n",
       "      <td>2.580130</td>\n",
       "      <td>-0.365617</td>\n",
       "      <td>0.849760</td>\n",
       "      <td>-1.017621</td>\n",
       "      <td>-0.146502</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 119 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         x0        x1       x10       x11       x12       x13       x14  \\\n",
       "0 -0.075227  0.431298 -0.102254  1.120197  0.260087 -1.199619 -2.450272   \n",
       "1  0.626326 -0.229215  0.293587  0.199684  0.331408  0.632379 -0.225701   \n",
       "2 -1.348959  0.400672 -0.934314 -1.947026 -0.016606  0.222856 -1.347693   \n",
       "3  0.368056 -0.034100 -0.384268  0.119804 -0.373455 -1.815058 -0.902405   \n",
       "4  0.168766  1.091534  2.082734 -0.515513  2.155874  2.580130 -0.365617   \n",
       "\n",
       "        x15       x16       x17  ...  x20_May  x20_Nov  x20_Oct  x20_Sept  \\\n",
       "0 -1.659281  1.102589  2.012662  ...      0.0      0.0      0.0       0.0   \n",
       "1 -0.157557 -0.516379 -0.119575  ...      0.0      0.0      0.0       0.0   \n",
       "2 -0.992515 -1.378931 -2.035756  ...      0.0      0.0      0.0       0.0   \n",
       "3 -0.288641 -0.613191 -1.453367  ...      0.0      0.0      0.0       0.0   \n",
       "4  0.849760 -1.017621 -0.146502  ...      0.0      0.0      0.0       0.0   \n",
       "\n",
       "   x27_Afternoon  x27_Evening  x27_Morning  x27_Night  x49_False  x49_True  \n",
       "0            1.0          0.0          0.0        0.0        1.0       0.0  \n",
       "1            1.0          0.0          0.0        0.0        1.0       0.0  \n",
       "2            0.0          1.0          0.0        0.0        1.0       0.0  \n",
       "3            1.0          0.0          0.0        0.0        0.0       1.0  \n",
       "4            0.0          1.0          0.0        0.0        1.0       0.0  \n",
       "\n",
       "[5 rows x 119 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numerical_cols = historical_df_without_y.select_dtypes(include=['int64']).columns\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='median')),\n",
    "    ('scaler', StandardScaler())])\n",
    "\n",
    "remaining_categorical_cols = ['x6', 'x20', 'x27', 'x49']\n",
    "numerical_cols = historical_df_without_y.columns.difference(remaining_categorical_cols + ['y'])\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "        ('imputer', SimpleImputer(strategy='most_frequent')),\n",
    "        ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', numeric_transformer, numerical_cols),\n",
    "        ('cat', categorical_transformer, remaining_categorical_cols)\n",
    "        ])\n",
    "\n",
    "preprocessor.fit(historical_df_without_y)\n",
    "historical_df_without_y = preprocessor.transform(historical_df_without_y)\n",
    "feature_names = list(numerical_cols) + \\\n",
    "                list(preprocessor.named_transformers_['cat'].named_steps['onehot'].get_feature_names_out(remaining_categorical_cols))\n",
    "\n",
    "\n",
    "historical_df_without_y = pd.DataFrame(historical_df_without_y, columns=feature_names)\n",
    "historical_df_without_y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.945125\n",
       "1    0.054875\n",
       "Name: y, dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "historical_df['y'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dataset shows a significant imbalance in the distribution of the target variable `y`, where:\n",
    "\n",
    "#### Strategy to Mitigate Imbalance\n",
    "To address this imbalance and improve the model's performance on minority class predictions, we will implement the following strategies:\n",
    "\n",
    "1. **Oversampling the Minority Class**:\n",
    "   - We will use **SMOTE** (Synthetic Minority Over-sampling Technique) to artificially increase the number of samples in the minority class (`1`). SMOTE generates synthetic samples from the minority class by interpolating existing samples, thus providing a more balanced dataset.\n",
    "\n",
    "2. **Undersampling the Majority Class**:\n",
    "   - Alongside oversampling, we will also apply **RandomUnderSampler** to reduce the number of samples in the majority class (`0`). This method randomly discards samples from the majority class to prevent its overwhelming influence during model training.\n",
    "\n",
    "3. **Experimenting with Different Ratios**:\n",
    "   - We will experiment with different oversampling and undersampling ratios to find the optimal balance that maximizes the model's performance, especially in terms of precision and recall for the minority class.\n",
    "   Ratios that will be tested: (10,90) , (20,80) , (30,70) , (40,60) , (50,50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Models to be Tested\n",
    "\n",
    "1. Support Vector Machine (SVM)\n",
    "\n",
    "2. Logistic Regression\n",
    "\n",
    "3. XGBoost\n",
    "\n",
    "4. Random Forest\n",
    "\n",
    "5. K-Nearest Neighbors (KNN):\n",
    "   - We test multiple values of \\( k \\) (e.g., 1, 5, 10, 15, 20) to determine the optimal number of neighbors that balances sensitivity and specificity.\n",
    "\n",
    "#### Evaluation Process\n",
    "\n",
    "- **Data Splitting**: The dataset is divided into 70% training and 30% testing, ensuring models are tested against unseen data.\n",
    "- **Model Fitting and Prediction**: Each model is trained on resampled training data and evaluated on the test set.\n",
    "- **Performance Metrics**: Accuracy, precision, recall, and F1-score are calculated for each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Overall Model: xgboost\n",
      "Ratio: (10, 90)\n",
      "Accuracy: 0.97\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98      2266\n",
      "           1       0.84      0.57      0.68       134\n",
      "\n",
      "    accuracy                           0.97      2400\n",
      "   macro avg       0.91      0.78      0.83      2400\n",
      "weighted avg       0.97      0.97      0.97      2400\n",
      "\n"
     ]
    }
   ],
   "source": [
    "models = [\n",
    "    {'name': 'svc', 'model': SVC()},\n",
    "    {'name': 'logistic regression', 'model': LogisticRegression(max_iter=1000)},\n",
    "    {'name': 'xgboost', 'model': XGBClassifier(use_label_encoder=False, eval_metric='logloss')},\n",
    "    {'name': 'random forest', 'model': RandomForestClassifier()}\n",
    "] + [{'name': f'knn, k={k}', 'model': KNeighborsClassifier(n_neighbors=k)} for k in range(1, 21, 4)]\n",
    "\n",
    "ratios = [(i, 100 - i) for i in range(10, 60, 10)]  # Format (minority, majority)\n",
    "\n",
    "x = historical_df_without_y\n",
    "y = historical_df['y']\n",
    "\n",
    "# Split data into training and testing\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42)\n",
    "\n",
    "best_overall_model = None\n",
    "max_overall_accuracy = 0\n",
    "\n",
    "for ratio in ratios:\n",
    "    strategy = {0: int(len(y_train) * ratio[1] / 100), 1: int(len(y_train) * ratio[0] / 100)}\n",
    "    pipeline = ImbPipeline([\n",
    "        ('oversample', SMOTE(sampling_strategy=strategy[1] / strategy[0])),\n",
    "        ('undersample', RandomUnderSampler(sampling_strategy=strategy)),\n",
    "    ])\n",
    "\n",
    "    # Resample the training data\n",
    "    X_resampled, y_resampled = pipeline.fit_resample(x_train, y_train)\n",
    "\n",
    "    for model_info in models:\n",
    "        model = model_info['model']\n",
    "        model_name = model_info['name']\n",
    "        model.fit(X_resampled, y_resampled)\n",
    "        y_pred = model.predict(x_test)\n",
    "        accuracy = accuracy_score(y_test, y_pred)\n",
    "        \n",
    "        if accuracy > max_overall_accuracy:\n",
    "            max_overall_accuracy = accuracy\n",
    "            best_overall_model = {\n",
    "                'model': model,\n",
    "                'model_name': model_name,\n",
    "                'sampling_ratio': ratio,\n",
    "                'accuracy': accuracy,\n",
    "                'report': classification_report(y_test, y_pred)\n",
    "            }\n",
    "\n",
    "# Display the best overall model\n",
    "if best_overall_model:\n",
    "    print(f\"Best Overall Model: {best_overall_model['model_name']}\")\n",
    "    print(f\"Ratio: {best_overall_model['sampling_ratio']}\")\n",
    "    print(f\"Accuracy: {best_overall_model['accuracy']}\\n\")\n",
    "    print(best_overall_model['report'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best Model has been evaluated to be xgboost with the (10,90) ratio."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To ensure consistency and accuracy in our predictions, we will apply the same preprocessing steps to the current transaction data as those used on the historical data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1896\n",
       "1     104\n",
       "dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_ratio = best_overall_model['sampling_ratio']\n",
    "strategy = {0: int(len(y_train) * best_ratio[1] / 100), 1: int(len(y_train) * best_ratio[0] / 100)}\n",
    "\n",
    "pipeline = ImbPipeline([\n",
    "    ('oversample', SMOTE(sampling_strategy=strategy[1] / strategy[0])),\n",
    "    ('undersample', RandomUnderSampler(sampling_strategy=strategy)),\n",
    "])\n",
    "\n",
    "current_df['x5'] = current_df['x5'].str.rstrip('%').astype('float') / 100.0\n",
    "current_df['x57'] = current_df['x57'].replace('[\\$,]', '', regex=True).astype('float')\n",
    "\n",
    "numerical_cols = current_df.select_dtypes(include=['int64']).columns\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='median')),\n",
    "    ('scaler', StandardScaler())])\n",
    "\n",
    "remaining_categorical_cols = ['x6', 'x20', 'x27', 'x49']\n",
    "numerical_cols = current_df.columns.difference(remaining_categorical_cols + ['y'])\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "        ('imputer', SimpleImputer(strategy='most_frequent')),\n",
    "        ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', numeric_transformer, numerical_cols),\n",
    "        ('cat', categorical_transformer, remaining_categorical_cols)\n",
    "        ])\n",
    "\n",
    "preprocessor.fit(current_df)\n",
    "transformed_current_df = preprocessor.transform(current_df)\n",
    "\n",
    "feature_names = list(numerical_cols) + \\\n",
    "                list(preprocessor.named_transformers_['cat'].named_steps['onehot'].get_feature_names_out(remaining_categorical_cols))\n",
    "transformed_current_df = pd.DataFrame(transformed_current_df, columns=feature_names)\n",
    "transformed_current_df = preprocessor.transform(current_df)\n",
    "\n",
    "# Now proceed to predict using the trained model\n",
    "model = best_overall_model['model'].fit(X_resampled, y_resampled)\n",
    "y_pred = model.predict(transformed_current_df)\n",
    "\n",
    "pd.DataFrame(y_pred).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_df = pd.DataFrame(y_pred, columns=['predicted y'])\n",
    "\n",
    "predictions_df.to_csv('./current_transaction_predictions.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}