Stacking Regressor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import random\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import StackingRegressor\n",
    "from sklearn import linear_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold, GridSearchCV, cross_validate, ParameterGrid\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.base import clone\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.svm import SVR, LinearSVR, NuSVR\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor, plot_tree\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor, AdaBoostRegressor\n",
    "from sklearn.linear_model import SGDRegressor, LogisticRegressionCV, RidgeCV\n",
    "\n",
    "from sklearn import linear_model, ensemble\n",
    "\n",
    "from sklearn.ensemble import StackingRegressor, VotingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>BsmtUnfSF</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleType_ConLw</th>\n",
       "      <th>SaleType_New</th>\n",
       "      <th>SaleType_Oth</th>\n",
       "      <th>SaleType_WD</th>\n",
       "      <th>SaleCondition_Abnorml</th>\n",
       "      <th>SaleCondition_AdjLand</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "      <th>Id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.129931</td>\n",
       "      <td>-0.155586</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.147933</td>\n",
       "      <td>0.929322</td>\n",
       "      <td>1.246171</td>\n",
       "      <td>0.794646</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-1.050480</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.553114</td>\n",
       "      <td>0.142585</td>\n",
       "      <td>-0.017057</td>\n",
       "      <td>1.934108</td>\n",
       "      <td>-0.010675</td>\n",
       "      <td>-0.592154</td>\n",
       "      <td>-0.768637</td>\n",
       "      <td>1.210991</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-0.574229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.015456</td>\n",
       "      <td>0.502563</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.051643</td>\n",
       "      <td>0.857357</td>\n",
       "      <td>1.108725</td>\n",
       "      <td>0.363497</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-0.106155</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.381843</td>\n",
       "      <td>0.130448</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>-1.620217</td>\n",
       "      <td>-0.822258</td>\n",
       "      <td>-0.768637</td>\n",
       "      <td>-0.375798</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>0.180128</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.720013</td>\n",
       "      <td>1.009159</td>\n",
       "      <td>1.322129</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.004078</td>\n",
       "      <td>0.716743</td>\n",
       "      <td>1.585134</td>\n",
       "      <td>0.704265</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>0.049879</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 128 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LotFrontage   LotArea  OverallQual  OverallCond  YearBuilt  YearRemodAdd  \\\n",
       "0    -0.129931 -0.155586     0.672280    -0.453217   1.147933      0.929322   \n",
       "1     0.553114  0.142585    -0.017057     1.934108  -0.010675     -0.592154   \n",
       "2     0.015456  0.502563     0.672280    -0.453217   1.051643      0.857357   \n",
       "3    -0.381843  0.130448     0.672280    -0.453217  -1.620217     -0.822258   \n",
       "4     0.720013  1.009159     1.322129    -0.453217   1.004078      0.716743   \n",
       "\n",
       "   MasVnrArea  BsmtFinSF1  BsmtFinSF2  BsmtUnfSF  ...  SaleType_ConLw  \\\n",
       "0    1.246171    0.794646   -0.365991  -1.050480  ...             0.0   \n",
       "1   -0.768637    1.210991   -0.365991  -0.574229  ...             0.0   \n",
       "2    1.108725    0.363497   -0.365991  -0.106155  ...             0.0   \n",
       "3   -0.768637   -0.375798   -0.365991   0.180128  ...             0.0   \n",
       "4    1.585134    0.704265   -0.365991   0.049879  ...             0.0   \n",
       "\n",
       "   SaleType_New  SaleType_Oth  SaleType_WD  SaleCondition_Abnorml  \\\n",
       "0           0.0           0.0          1.0                    0.0   \n",
       "1           0.0           0.0          1.0                    0.0   \n",
       "2           0.0           0.0          1.0                    0.0   \n",
       "3           0.0           0.0          1.0                    1.0   \n",
       "4           0.0           0.0          1.0                    0.0   \n",
       "\n",
       "   SaleCondition_AdjLand  SaleCondition_Alloca  SaleCondition_Family  \\\n",
       "0                    0.0                   0.0                   0.0   \n",
       "1                    0.0                   0.0                   0.0   \n",
       "2                    0.0                   0.0                   0.0   \n",
       "3                    0.0                   0.0                   0.0   \n",
       "4                    0.0                   0.0                   0.0   \n",
       "\n",
       "   SaleCondition_Partial  Id  \n",
       "0                    0.0   1  \n",
       "1                    0.0   2  \n",
       "2                    0.0   3  \n",
       "3                    0.0   4  \n",
       "4                    0.0   5  \n",
       "\n",
       "[5 rows x 128 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loc_train = 'data\\\\train.csv'\n",
    "loc_test = 'data\\\\test.csv'\n",
    "\n",
    "target = 'SalePrice'\n",
    "evaluation = 'neg_mean_squared_log_error'\n",
    "\n",
    "raw_train_data = pd.read_csv(loc_train)\n",
    "y = raw_train_data[target]\n",
    "raw_train_data = raw_train_data.drop(target, axis=1)\n",
    "raw_test_data = pd.read_csv(loc_test)\n",
    "raw_train_data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare(raw_DF):\n",
    "    DF = raw_DF.copy()\n",
    "    \n",
    "    DF = DF.drop('Id', axis= 1)\n",
    "    \n",
    "    return DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>BsmtUnfSF</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleType_ConLw</th>\n",
       "      <th>SaleType_New</th>\n",
       "      <th>SaleType_Oth</th>\n",
       "      <th>SaleType_WD</th>\n",
       "      <th>SaleCondition_Abnorml</th>\n",
       "      <th>SaleCondition_AdjLand</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.129931</td>\n",
       "      <td>-0.155586</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.147933</td>\n",
       "      <td>0.929322</td>\n",
       "      <td>1.246171</td>\n",
       "      <td>0.794646</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-1.050480</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.553114</td>\n",
       "      <td>0.142585</td>\n",
       "      <td>-0.017057</td>\n",
       "      <td>1.934108</td>\n",
       "      <td>-0.010675</td>\n",
       "      <td>-0.592154</td>\n",
       "      <td>-0.768637</td>\n",
       "      <td>1.210991</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-0.574229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.015456</td>\n",
       "      <td>0.502563</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.051643</td>\n",
       "      <td>0.857357</td>\n",
       "      <td>1.108725</td>\n",
       "      <td>0.363497</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>-0.106155</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.381843</td>\n",
       "      <td>0.130448</td>\n",
       "      <td>0.672280</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>-1.620217</td>\n",
       "      <td>-0.822258</td>\n",
       "      <td>-0.768637</td>\n",
       "      <td>-0.375798</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>0.180128</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.720013</td>\n",
       "      <td>1.009159</td>\n",
       "      <td>1.322129</td>\n",
       "      <td>-0.453217</td>\n",
       "      <td>1.004078</td>\n",
       "      <td>0.716743</td>\n",
       "      <td>1.585134</td>\n",
       "      <td>0.704265</td>\n",
       "      <td>-0.365991</td>\n",
       "      <td>0.049879</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 128 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LotFrontage   LotArea  OverallQual  OverallCond  YearBuilt  YearRemodAdd  \\\n",
       "0    -0.129931 -0.155586     0.672280    -0.453217   1.147933      0.929322   \n",
       "1     0.553114  0.142585    -0.017057     1.934108  -0.010675     -0.592154   \n",
       "2     0.015456  0.502563     0.672280    -0.453217   1.051643      0.857357   \n",
       "3    -0.381843  0.130448     0.672280    -0.453217  -1.620217     -0.822258   \n",
       "4     0.720013  1.009159     1.322129    -0.453217   1.004078      0.716743   \n",
       "\n",
       "   MasVnrArea  BsmtFinSF1  BsmtFinSF2  BsmtUnfSF  ...  SaleType_ConLw  \\\n",
       "0    1.246171    0.794646   -0.365991  -1.050480  ...             0.0   \n",
       "1   -0.768637    1.210991   -0.365991  -0.574229  ...             0.0   \n",
       "2    1.108725    0.363497   -0.365991  -0.106155  ...             0.0   \n",
       "3   -0.768637   -0.375798   -0.365991   0.180128  ...             0.0   \n",
       "4    1.585134    0.704265   -0.365991   0.049879  ...             0.0   \n",
       "\n",
       "   SaleType_New  SaleType_Oth  SaleType_WD  SaleCondition_Abnorml  \\\n",
       "0           0.0           0.0          1.0                    0.0   \n",
       "1           0.0           0.0          1.0                    0.0   \n",
       "2           0.0           0.0          1.0                    0.0   \n",
       "3           0.0           0.0          1.0                    1.0   \n",
       "4           0.0           0.0          1.0                    0.0   \n",
       "\n",
       "   SaleCondition_AdjLand  SaleCondition_Alloca  SaleCondition_Family  \\\n",
       "0                    0.0                   0.0                   0.0   \n",
       "1                    0.0                   0.0                   0.0   \n",
       "2                    0.0                   0.0                   0.0   \n",
       "3                    0.0                   0.0                   0.0   \n",
       "4                    0.0                   0.0                   0.0   \n",
       "\n",
       "   SaleCondition_Partial       y  \n",
       "0                    0.0  208500  \n",
       "1                    0.0  181500  \n",
       "2                    0.0  223500  \n",
       "3                    0.0  140000  \n",
       "4                    0.0  250000  \n",
       "\n",
       "[5 rows x 128 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = prepare(raw_train_data)\n",
    "test_X = prepare(raw_test_data)\n",
    "\n",
    "F = train_X.copy()\n",
    "F['y'] = y\n",
    "F.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.log(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "GBR = GradientBoostingRegressor(n_estimators=8000, \n",
    "                               loss= 'huber')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://scikit-learn.org/stable/supervised_learning.html\n",
    "\n",
    "Regressors_small = [\n",
    "    GradientBoostingRegressor(),\n",
    "    KNeighborsRegressor(), \n",
    "    RandomForestRegressor(),\n",
    "    ExtraTreesRegressor(),\n",
    "]\n",
    "\n",
    "Regressors_medium = [ \n",
    "    SVR(), \n",
    "    LinearSVR(), \n",
    "    NuSVR(),\n",
    "    linear_model.RidgeCV(),\n",
    "    linear_model.LarsCV(),\n",
    "    linear_model.LassoCV(),\n",
    "    linear_model.LassoLarsCV(),\n",
    "    linear_model.PassiveAggressiveRegressor(), \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameter tuning\n",
    "gbt_parameters = {\n",
    "    'n_estimators': [100, 300, 500, 700], \n",
    "    'loss': ['ls', 'lad', 'huber'], \n",
    "    'warm_start': [True]\n",
    "}\n",
    "\n",
    "knn_parameters = {\n",
    "    'n_neighbors': range(7, 12),\n",
    "    'weights': ['distance']\n",
    "}\n",
    "\n",
    "rf_parameters = {\n",
    "    'n_estimators': [100, 300, 500, 700], \n",
    "    'warm_start': [True]\n",
    "}\n",
    "\n",
    "et_parameters = {\n",
    "    'n_estimators': [100, 300, 500, 700], \n",
    "    'warm_start': [True]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbt_models = [GradientBoostingRegressor(**params) for params in ParameterGrid(gbt_parameters)]\n",
    "knn_models = [KNeighborsRegressor(**params) for params in ParameterGrid(knn_parameters)]\n",
    "et_models = [ExtraTreesRegressor(**params) for params in ParameterGrid(et_parameters)]\n",
    "rf_models = [RandomForestRegressor(**params) for params in ParameterGrid(rf_parameters)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MLA = Regressors_small\n",
    "MLA = gbt_models + knn_models + et_models + rf_models\n",
    "MLA = MLA + Regressors_medium\n",
    "len(MLA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('GradientBoostingRegressor0', GradientBoostingRegressor(warm_start=True)),\n",
       " ('GradientBoostingRegressor1',\n",
       "  GradientBoostingRegressor(n_estimators=300, warm_start=True)),\n",
       " ('GradientBoostingRegressor2',\n",
       "  GradientBoostingRegressor(n_estimators=500, warm_start=True)),\n",
       " ('GradientBoostingRegressor3',\n",
       "  GradientBoostingRegressor(n_estimators=700, warm_start=True)),\n",
       " ('GradientBoostingRegressor4',\n",
       "  GradientBoostingRegressor(loss='lad', warm_start=True))]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of the Classifiers to be stacked\n",
    "\n",
    "Stack_estimators = [\n",
    "    (a.__class__.__name__+str(i), a) for i, a in enumerate(MLA)\n",
    "]\n",
    "Stack_estimators[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingRegressor(estimators=[('GradientBoostingRegressor0',\n",
       "                             GradientBoostingRegressor(warm_start=True)),\n",
       "                            ('GradientBoostingRegressor1',\n",
       "                             GradientBoostingRegressor(n_estimators=300,\n",
       "                                                       warm_start=True)),\n",
       "                            ('GradientBoostingRegressor2',\n",
       "                             GradientBoostingRegressor(n_estimators=500,\n",
       "                                                       warm_start=True)),\n",
       "                            ('GradientBoostingRegressor3',\n",
       "                             GradientBoostingRegressor(n_estimato...\n",
       "                             RandomForestRegressor(n_estimators=300,\n",
       "                                                   warm_start=True)),\n",
       "                            ('RandomForestRegressor23',\n",
       "                             RandomForestRegressor(n_estimators=500,\n",
       "                                                   warm_start=True)),\n",
       "                            ('RandomForestRegressor24',\n",
       "                             RandomForestRegressor(n_estimators=700,\n",
       "                                                   warm_start=True)),\n",
       "                            ('SVR25', SVR()), ('LinearSVR26', LinearSVR()),\n",
       "                            ('NuSVR27', NuSVR()),\n",
       "                            ('RidgeCV28',\n",
       "                             RidgeCV(alphas=array([ 0.1,  1. , 10. ]))),\n",
       "                            ('LarsCV29', LarsCV()), ...])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voter = VotingRegressor(Stack_estimators)\n",
    "voter.fit(train_X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mla = [(f'gbr{i}', GradientBoostingRegressor(loss=random.choice(['huber', 'ls'])))\n",
    "#                                                           for i in range(100)]\n",
    "# mla[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Meta_models = [\n",
    "    linear_model.RidgeCV(),\n",
    "    linear_model.LassoCV(),\n",
    "    linear_model.LassoLarsCV(), \n",
    "    linear_model.ElasticNetCV()\n",
    "]\n",
    "names = ['ridge', 'lasso', 'lassolars', 'elasticnet']\n",
    "stacker = [StackingRegressor(Stack_estimators, final_estimator=mm) for mm in Meta_models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461</td>\n",
       "      <td>121610.835374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1462</td>\n",
       "      <td>154944.092279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1463</td>\n",
       "      <td>185676.670432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1464</td>\n",
       "      <td>191882.093158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1465</td>\n",
       "      <td>186434.217000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Id      SalePrice\n",
       "0  1461  121610.835374\n",
       "1  1462  154944.092279\n",
       "2  1463  185676.670432\n",
       "3  1464  191882.093158\n",
       "4  1465  186434.217000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voter_pred = np.exp(voter.predict(test_X))\n",
    "name = 'Voter'\n",
    "output = pd.DataFrame({'Id': raw_test_data.Id, target: voter_pred})\n",
    "output.to_csv(f'preds\\\\GBR\\\\{name}stack_7.csv', index = False)\n",
    "output.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "for est in stacker:\n",
    "    est.fit(train_X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461</td>\n",
       "      <td>118710.012199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1462</td>\n",
       "      <td>157432.703675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1463</td>\n",
       "      <td>187971.328563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1464</td>\n",
       "      <td>198109.777823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1465</td>\n",
       "      <td>189647.865404</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Id      SalePrice\n",
       "0  1461  118710.012199\n",
       "1  1462  157432.703675\n",
       "2  1463  187971.328563\n",
       "3  1464  198109.777823\n",
       "4  1465  189647.865404"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = [np.exp(est.predict(test_X)) for est in stacker]\n",
    "\n",
    "for name, pred in zip(names, predictions):\n",
    "    output = pd.DataFrame({'Id': raw_test_data.Id, target: pred})\n",
    "    output.to_csv(f'preds\\\\GBR\\\\{name}stack_5.csv', index = False)\n",
    "output.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.058257844560175756\n",
      "0.05487236697735753\n",
      "0.23899263798318296\n",
      "0.05822802268167313\n",
      "0.05821451732263694\n"
     ]
    }
   ],
   "source": [
    "for est in stacker:\n",
    "    print (sklearn.metrics.mean_squared_error(y, est.predict(train_X))**0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit",
   "language": "python",
   "name": "python37464bitb122d7ed60ea485982511a1bcddbd83a"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}