5-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rOafwFRXG7fb"
   },
   "source": [
    "# 교차 검증과 그리드 서치"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6YuimmymG7fi"
   },
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/rickiepark/hg-mldl/blob/master/5-2.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />구글 코랩에서 실행하기</a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dVNF7yZjyvoO"
   },
   "source": [
    "## 검증 세트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "banlvMA6RfnM"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "wine = pd.read_csv('https://bit.ly/wine_csv_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "abR6QA7qRoKl"
   },
   "outputs": [],
   "source": [
    "data = wine[['alcohol', 'sugar', 'pH']].to_numpy()\n",
    "target = wine['class'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "auLnVXyMRoeb"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_input, test_input, train_target, test_target = train_test_split(\n",
    "    data, target, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "E-yV4cCXRqNK"
   },
   "outputs": [],
   "source": [
    "sub_input, val_input, sub_target, val_target = train_test_split(\n",
    "    train_input, train_target, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "k29hKbw4R7Ki",
    "outputId": "87ccf731-e87c-42d4-b12c-419aa55e7221"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4157, 3) (1040, 3)\n"
     ]
    }
   ],
   "source": [
    "print(sub_input.shape, val_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4VQz-UZ2SeLq",
    "outputId": "d1c896a3-eab5-4437-961e-194b04397f78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9971133028626413\n",
      "0.864423076923077\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dt = DecisionTreeClassifier(random_state=42)\n",
    "dt.fit(sub_input, sub_target)\n",
    "\n",
    "print(dt.score(sub_input, sub_target))\n",
    "print(dt.score(val_input, val_target))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Z4gRXnK6y2Pt"
   },
   "source": [
    "## 교차 검증"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_J3LId-vSmNH",
    "outputId": "e240b691-821a-4e62-90fa-b329ad8f0821"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'fit_time': array([0.00761032, 0.0069797 , 0.00732017, 0.007411  , 0.00687742]), 'score_time': array([0.00072193, 0.00064898, 0.0006237 , 0.00067067, 0.00064206]), 'test_score': array([0.86923077, 0.84615385, 0.87680462, 0.84889317, 0.83541867])}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_validate\n",
    "\n",
    "scores = cross_validate(dt, train_input, train_target)\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Yp3aagOoTHsO",
    "outputId": "8d4273bf-9e62-43f0-ef4e-981dfb8450ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.855300214703487\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "print(np.mean(scores['test_score']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0tQyaG0576Vn",
    "outputId": "2b6b230d-a1ce-4d4c-df04-efaa3c2a6645"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.855300214703487\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "scores = cross_validate(dt, train_input, train_target, cv=StratifiedKFold())\n",
    "print(np.mean(scores['test_score']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1BmP_OTT_agM",
    "outputId": "6ceb7677-b76b-416e-e344-9d1c8d4452e8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8574181117533719\n"
     ]
    }
   ],
   "source": [
    "splitter = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\n",
    "scores = cross_validate(dt, train_input, train_target, cv=splitter)\n",
    "print(np.mean(scores['test_score']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q21W8RsqDsDV"
   },
   "source": [
    "## 하이퍼파라미터 튜닝"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "S8pqss8onjR5"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'min_impurity_decrease': [0.0001, 0.0002, 0.0003, 0.0004, 0.0005]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "79MymJqxTu0P"
   },
   "outputs": [],
   "source": [
    "gs = GridSearchCV(DecisionTreeClassifier(random_state=42), params, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tKAlTabkU-Lz",
    "outputId": "b7d6c052-42e6-4b01-b6fb-bd78b6f523d3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=DecisionTreeClassifier(random_state=42), n_jobs=-1,\n",
       "             param_grid={'min_impurity_decrease': [0.0001, 0.0002, 0.0003,\n",
       "                                                   0.0004, 0.0005]})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.fit(train_input, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q6iX3vH-VeEb",
    "outputId": "cede4267-3135-4644-f625-7d37a894545d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9615162593804117\n"
     ]
    }
   ],
   "source": [
    "dt = gs.best_estimator_\n",
    "print(dt.score(train_input, train_target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lIzod3BwVHq-",
    "outputId": "2f84cffc-e7c7-469e-b0cb-774121cdca71"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min_impurity_decrease': 0.0001}\n"
     ]
    }
   ],
   "source": [
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0xfQswiui4Tr",
    "outputId": "404175ae-7e86-48f3-8f05-10ce223bc557"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.86819297 0.86453617 0.86492226 0.86780891 0.86761605]\n"
     ]
    }
   ],
   "source": [
    "print(gs.cv_results_['mean_test_score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Rwg2aSyEVO17",
    "outputId": "625cdf68-2172-47fa-ff67-51f71a23f8ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min_impurity_decrease': 0.0001}\n"
     ]
    }
   ],
   "source": [
    "best_index = np.argmax(gs.cv_results_['mean_test_score'])\n",
    "print(gs.cv_results_['params'][best_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "8jHxZ7XmVU11"
   },
   "outputs": [],
   "source": [
    "params = {'min_impurity_decrease': np.arange(0.0001, 0.001, 0.0001),\n",
    "          'max_depth': range(5, 20, 1),\n",
    "          'min_samples_split': range(2, 100, 10)\n",
    "          }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KnP3GA6MVsVH",
    "outputId": "04a03cbe-ae4d-4dad-b55b-5ba610e51bbd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=DecisionTreeClassifier(random_state=42), n_jobs=-1,\n",
       "             param_grid={'max_depth': range(5, 20),\n",
       "                         'min_impurity_decrease': array([0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008,\n",
       "       0.0009]),\n",
       "                         'min_samples_split': range(2, 100, 10)})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs = GridSearchCV(DecisionTreeClassifier(random_state=42), params, n_jobs=-1)\n",
    "gs.fit(train_input, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qi9-O_VGV0Ho",
    "outputId": "521159e8-3b2a-4cd1-dc00-95dbed67db89"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 14, 'min_impurity_decrease': 0.0004, 'min_samples_split': 12}\n"
     ]
    }
   ],
   "source": [
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZnJjLATAV2Sq",
    "outputId": "1c432df8-5e44-4fdf-c9db-0f53ff366c7a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8683865773302731\n"
     ]
    }
   ],
   "source": [
    "print(np.max(gs.cv_results_['mean_test_score']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d0k9DQTNlaD6"
   },
   "source": [
    "### 랜덤 서치"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "_T9KTEk1GBcY"
   },
   "outputs": [],
   "source": [
    "from scipy.stats import uniform, randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fd0UJpCGGDhz",
    "outputId": "f5a4b4ce-34d1-4765-c047-a73b64eb61e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 6, 1, 7, 4, 1, 4, 9, 7, 0])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgen = randint(0, 10)\n",
    "rgen.rvs(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ch3zTUohIJR6",
    "outputId": "c28a1a61-e438-4764-ce9e-51f84a1820b9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),\n",
       " array([100, 114, 115,  88, 108,  80, 104,  91, 101,  99]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(rgen.rvs(1000), return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bGhshTn0IjkI",
    "outputId": "b9bf8ec0-5f11-4074-a849-f6bb0e8e2035"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.51779717, 0.98001995, 0.74399202, 0.42051715, 0.10599781,\n",
       "       0.03354555, 0.20682412, 0.96337699, 0.43563022, 0.78230147])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ugen = uniform(0, 1)\n",
    "ugen.rvs(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "irDX9e6WYTIH"
   },
   "outputs": [],
   "source": [
    "params = {'min_impurity_decrease': uniform(0.0001, 0.001),\n",
    "          'max_depth': randint(20, 50),\n",
    "          'min_samples_split': randint(2, 25),\n",
    "          'min_samples_leaf': randint(1, 25),\n",
    "          }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Wc4OIingWQCK",
    "outputId": "4fa423e7-7a7b-4486-ad75-fa0560df4e2c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(estimator=DecisionTreeClassifier(random_state=42),\n",
       "                   n_iter=100, n_jobs=-1,\n",
       "                   param_distributions={'max_depth': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f8639482668>,\n",
       "                                        'min_impurity_decrease': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f8639482f28>,\n",
       "                                        'min_samples_leaf': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f86394d09b0>,\n",
       "                                        'min_samples_split': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f86394d0550>},\n",
       "                   random_state=42)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "gs = RandomizedSearchCV(DecisionTreeClassifier(random_state=42), params, \n",
    "                        n_iter=100, n_jobs=-1, random_state=42)\n",
    "gs.fit(train_input, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "p7IbsGH3ZSv-",
    "outputId": "efeb14d9-e26b-4c44-a8bc-8165b0e9f79a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 39, 'min_impurity_decrease': 0.00034102546602601173, 'min_samples_leaf': 7, 'min_samples_split': 13}\n"
     ]
    }
   ],
   "source": [
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dYI3HwMQbtnr",
    "outputId": "6f07592d-c048-4b75-963b-4cb41ffcd170"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8695428296438884\n"
     ]
    }
   ],
   "source": [
    "print(np.max(gs.cv_results_['mean_test_score']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3QV7yRpidByf",
    "outputId": "23c87a7e-18ef-4260-ec19-5ca5ba1e8e0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.86\n"
     ]
    }
   ],
   "source": [
    "dt = gs.best_estimator_\n",
    "\n",
    "print(dt.score(test_input, test_target))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cA42IsMdhgE7"
   },
   "source": [
    "## 확인문제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8qxg36iThiUm",
    "outputId": "a1d5e027-3789-46c5-c611-a5752904b01d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(estimator=DecisionTreeClassifier(random_state=42,\n",
       "                                                    splitter='random'),\n",
       "                   n_iter=100, n_jobs=-1,\n",
       "                   param_distributions={'max_depth': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f8639482668>,\n",
       "                                        'min_impurity_decrease': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f8639482f28>,\n",
       "                                        'min_samples_leaf': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f86394d09b0>,\n",
       "                                        'min_samples_split': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f86394d0550>},\n",
       "                   random_state=42)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs = RandomizedSearchCV(DecisionTreeClassifier(splitter='random', random_state=42), params, \n",
    "                        n_iter=100, n_jobs=-1, random_state=42)\n",
    "gs.fit(train_input, train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CMZ4UE8ihqwg",
    "outputId": "17a01931-779b-4d67-9c91-fbf0d1459993"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 43, 'min_impurity_decrease': 0.00011407982271508446, 'min_samples_leaf': 19, 'min_samples_split': 18}\n",
      "0.8458726956392981\n",
      "0.786923076923077\n"
     ]
    }
   ],
   "source": [
    "print(gs.best_params_)\n",
    "print(np.max(gs.cv_results_['mean_test_score']))\n",
    "\n",
    "dt = gs.best_estimator_\n",
    "print(dt.score(test_input, test_target))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "5-2 교차 검증과 그리드 서치.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}