Apply_Grid_Search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from pandas.core.common import SettingWithCopyWarning\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import make_scorer\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('train/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Day</th>\n",
       "      <th>Hour</th>\n",
       "      <th>Minute</th>\n",
       "      <th>DHI</th>\n",
       "      <th>DNI</th>\n",
       "      <th>WS</th>\n",
       "      <th>RH</th>\n",
       "      <th>T</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>69.08</td>\n",
       "      <td>-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>69.06</td>\n",
       "      <td>-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>71.78</td>\n",
       "      <td>-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>71.75</td>\n",
       "      <td>-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>75.20</td>\n",
       "      <td>-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52555</th>\n",
       "      <td>1094</td>\n",
       "      <td>21</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>70.70</td>\n",
       "      <td>-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52556</th>\n",
       "      <td>1094</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>66.79</td>\n",
       "      <td>-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52557</th>\n",
       "      <td>1094</td>\n",
       "      <td>22</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>66.78</td>\n",
       "      <td>-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52558</th>\n",
       "      <td>1094</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.1</td>\n",
       "      <td>67.72</td>\n",
       "      <td>-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52559</th>\n",
       "      <td>1094</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.1</td>\n",
       "      <td>67.70</td>\n",
       "      <td>-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>52560 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Day  Hour  Minute  DHI  DNI   WS     RH   T  TARGET\n",
       "0         0     0       0    0    0  1.5  69.08 -12     0.0\n",
       "1         0     0      30    0    0  1.5  69.06 -12     0.0\n",
       "2         0     1       0    0    0  1.6  71.78 -12     0.0\n",
       "3         0     1      30    0    0  1.6  71.75 -12     0.0\n",
       "4         0     2       0    0    0  1.6  75.20 -12     0.0\n",
       "...     ...   ...     ...  ...  ...  ...    ...  ..     ...\n",
       "52555  1094    21      30    0    0  2.4  70.70  -4     0.0\n",
       "52556  1094    22       0    0    0  2.4  66.79  -4     0.0\n",
       "52557  1094    22      30    0    0  2.2  66.78  -4     0.0\n",
       "52558  1094    23       0    0    0  2.1  67.72  -4     0.0\n",
       "52559  1094    23      30    0    0  2.1  67.70  -4     0.0\n",
       "\n",
       "[52560 rows x 9 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = df.drop(['Minute', 'Hour', 'Day', 'TARGET'], axis=1)\n",
    "y_train = df['TARGET']\n",
    "\n",
    "X_train = X_train[:-96]\n",
    "y_train = y_train[96:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling Grid_Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pinball_loss(y_true, y_pred, tau=0.5):\n",
    "    return np.nanmean((y_pred >= y_true) * (1 - tau) * (y_pred - y_true) + (y_pred < y_true) * tau * (y_true - y_pred))\n",
    "\n",
    "pinball_scorer = make_scorer(pinball_loss, greater_is_better=False)\n",
    "\n",
    "param = {\n",
    "    'max_depth':[5,6],\n",
    "    'n_estimators':[10000],\n",
    "    'colsample_bytree':[0.7],\n",
    "    'colsample_bylevel':[0.7],\n",
    "}\n",
    "model = xgb.XGBRegressor(max_depth=7)\n",
    "\n",
    "grid_search = GridSearchCV(estimator=model, param_grid=param, \n",
    "                           scoring=pinball_scorer,\n",
    "                           n_jobs=-1)\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "print(grid_search.best_params_)\n",
    "print(grid_search.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pinball Loss score:  2.017757684194703\n"
     ]
    }
   ],
   "source": [
    "pred_train = grid_search.predict(X_train)\n",
    "\n",
    "print('Pinball Loss score: ', pinball_loss(y_train, pred_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('test/0.csv')\n",
    "\n",
    "X_test = X_test.drop(['Minute', 'Hour', 'Day', 'TARGET'], axis=1)\n",
    "X_test = X_test[336-96:]\n",
    "\n",
    "pred_test = grid_search.predict(X_test)\n",
    "\n",
    "for i in range(len(pred_test)):\n",
    "    if pred_test[i] < 0 :\n",
    "        pred_test[i] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pinball_loss(y_true, y_pred, tau):\n",
    "    return np.nanmean((y_pred >= y_true) * (1 - tau) * (y_pred - y_true) + (y_pred < y_true) * tau * (y_true - y_pred))\n",
    "\n",
    "def build_model(tau):\n",
    "    pinball_scorer = make_scorer(pinball_loss, greater_is_better=False)\n",
    "    \n",
    "    print(\"Tau : \", tau)    \n",
    "    \n",
    "    param = {\n",
    "        'max_depth':[6],\n",
    "        'n_estimators':[10000],\n",
    "        'colsample_bytree':[0.7],\n",
    "        'colsample_bylevel':[0.7],\n",
    "    }\n",
    "    model = xgb.XGBRegressor(max_depth=7)\n",
    "\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param, \n",
    "                               scoring=pinball_scorer,\n",
    "                               n_jobs=-1)\n",
    "\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(grid_search.best_params_)\n",
    "    print(grid_search.best_estimator_)\n",
    "    \n",
    "    return grid_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_predict(model):     \n",
    "    dfs = []\n",
    "    outs = []\n",
    "    \n",
    "    X_test = pd.read_csv('test/0.csv')\n",
    "    X_test = X_test.drop(['Minute', 'Hour', 'Day', 'TARGET'], axis=1)\n",
    "    X_test = X_test[336-96:]\n",
    "\n",
    "    for i in range(81):\n",
    "        data = pd.read_csv(f\"test/{i}.csv\")\n",
    "        data = data.drop(['Minute', 'Hour', 'Day', 'TARGET'], axis=1)\n",
    "        data = data[336-96:]\n",
    "        \n",
    "        pred = model.predict(data)\n",
    "        \n",
    "        outs.append(pred)\n",
    "        \n",
    "    return np.array(outs).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tau :  0.1\n",
      "{'colsample_bylevel': 0.7, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.2\n",
      "{'colsample_bylevel': 0.7, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.3\n",
      "{'colsample_bylevel': 0.7, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.4\n",
      "{'colsample_bylevel': 0.7, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.5\n",
      "{'colsample_bylevel': 0.7, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.6\n",
      "{'colsample_bylevel': 1, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.7\n",
      "{'colsample_bylevel': 1, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.8\n",
      "{'colsample_bylevel': 1, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
      "Tau :  0.9\n",
      "{'colsample_bylevel': 1, 'colsample_bytree': 1, 'max_depth': 4, 'n_estimators': 300}\n",
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
      "             importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.300000012, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=300, n_jobs=8, num_parallel_tree=1, random_state=0,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None)\n"
     ]
    }
   ],
   "source": [
    "models = []\n",
    "taus = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]\n",
    "\n",
    "for i in range(len(taus)):\n",
    "    model = build_model(taus[i])\n",
    "    models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.simplefilter(action=\"ignore\", category=SettingWithCopyWarning)\n",
    "\n",
    "df_sample = pd.read_csv('sample_submission.csv')\n",
    "\n",
    "columns = ['q_0.1', 'q_0.2', 'q_0.3', 'q_0.4', 'q_0.5', 'q_0.6', 'q_0.7', 'q_0.8', 'q_0.9']\n",
    "\n",
    "for i in range(len(columns)):\n",
    "    out = do_predict(models[i])\n",
    "    df_sample[columns[i]] = out\n",
    "    \n",
    "df_sample.to_csv(f'XGBoost_Second_Try_Tiny_change.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for h in hiss:\n",
    "    plt.plot(h.history['loss'])\n",
    "    plt.plot(h.history['val_loss'])\n",
    "    plt.title('model loss')\n",
    "    plt.ylabel('loss')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.legend(['train', 'test'], loc='upper left')\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Dacon",
   "language": "python",
   "name": "dacon"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}