hyperparameter_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.space import Integer, Real\n",
    "\n",
    "def model_params(model_name):\n",
    "    # lgbm params\n",
    "    lgbm_params = {\n",
    "        \"boosting_type\": [\"gbdt\"],               \n",
    "        \"objective\": [\"multiclass\"],             \n",
    "        \"metric\": [\"multi_logloss\"],             \n",
    "        \"num_class\": [4],                        \n",
    "        \"num_leaves\": Integer(30, 100),          \n",
    "        \"learning_rate\": Real(0.01, 0.1, prior=\"log-uniform\"),  \n",
    "        \"n_estimators\": Integer(50, 500),       \n",
    "        \"random_state\": [42],                   \n",
    "        \"verbose\": [0]                          \n",
    "    }\n",
    "    xgb_params = {\n",
    "        \"objective\": [\"multi:softprob\"],                \n",
    "        \"num_class\": [4],                              \n",
    "        \"max_depth\": Integer(3, 10),                    \n",
    "        \"learning_rate\": Real(0.01, 0.1, prior=\"log-uniform\"),\n",
    "        \"n_estimators\": Integer(50, 200),\n",
    "        \"random_state\": [42],                          \n",
    "        \"verbosity\": [0]\n",
    "    }\n",
    "    rf_params = {\n",
    "        \"n_estimators\": Integer(50, 200),               \n",
    "        \"max_depth\": Integer(3, 20),                    \n",
    "        \"min_samples_split\": Integer(2, 20),            \n",
    "        \"min_samples_leaf\": Integer(1, 10),             \n",
    "        \"max_features\": [\"auto\", \"sqrt\", \"log2\"],       \n",
    "        \"bootstrap\": [True, False],                     \n",
    "        \"criterion\": [\"gini\", \"entropy\"],               \n",
    "        \"random_state\": [42],                          \n",
    "        \"n_jobs\": [1]                                  \n",
    "    }\n",
    "\n",
    "    catboost_params = {\n",
    "        \"iterations\": Integer(500, 1500),               \n",
    "        \"learning_rate\": Real(0.01, 0.2, prior=\"log-uniform\"),  \n",
    "        \"depth\": Integer(4, 10),                        \n",
    "        \"l2_leaf_reg\": Real(1, 10),                    \n",
    "        \"bagging_temperature\": Real(0.5, 2.0),         \n",
    "        \"random_seed\": [42],                           \n",
    "        \"verbose\": [100],                              \n",
    "        \"loss_function\": ['MultiClass'],               \n",
    "    }\n",
    "    \n",
    "    from utils.focal_loss import focal_loss_lgb\n",
    "    focal_loss = lambda x,y: focal_loss_lgb(x, y, 0.25, 2.0, 4)\n",
    "    fclgbm_params = {\n",
    "        \"num_class\":4,\n",
    "        \"objective\": focal_loss,\n",
    "        \"boosting_type\": \"gbdt\",\n",
    "        \"num_leaves\": Integer(30, 100),                 \n",
    "        \"learning_rate\": Real(0.01, 0.1, prior=\"log-uniform\"),  \n",
    "        \"n_estimators\": Integer(20, 100),               \n",
    "        \"random_state\": [42],                          \n",
    "        \"verbose\": [0],                                \n",
    "    }\n",
    "    svm_params = {\n",
    "        \"C\": Real(0.1, 10, prior=\"log-uniform\"),        \n",
    "        \"kernel\": [\"linear\", \"poly\", \"rbf\", \"sigmoid\"], \n",
    "        \"degree\": Integer(2, 5),                        \n",
    "        \"gamma\": [\"scale\", \"auto\"],                     \n",
    "        \"tol\": Real(1e-4, 1e-2),                       \n",
    "        \"max_iter\": [-1],\n",
    "        \"coef0\": 0.0,\n",
    "        \"shrinking\": True,\n",
    "        \"probability\": True,                       \n",
    "    }\n",
    "\n",
    "    if model_name == \"LGBM\":\n",
    "        return lgbm_params\n",
    "    elif model_name == \"FCLGBM\":\n",
    "        return fclgbm_params\n",
    "    elif model_name == \"XGB\":\n",
    "        return xgb_params\n",
    "    elif model_name == \"RF\":\n",
    "        return rf_params\n",
    "    elif model_name == \"CatBoost\":\n",
    "        return catboost_params\n",
    "    elif model_name == \"SVM\":\n",
    "        return svm_params\n",
    "    else:\n",
    "        print(\"Invalid model name. (Params)\")\n",
    "\n",
    "def hyperparameter_tuning(model_name, param_dist, X_train, y_train, method=\"grid\", n_iter=50):\n",
    "    # 모델 이름에 따라 해당 모델 객체를 가져오는 함수\n",
    "    def get_model(model_name):\n",
    "        if model_name == \"LGBM\":\n",
    "            from lightgbm import LGBMClassifier\n",
    "            return LGBMClassifier(**model_params(\"LGBM\"))\n",
    "        elif model_name == \"FCLGBM\":\n",
    "            from lightgbm import LGBMClassifier\n",
    "            return LGBMClassifier(**model_params(\"FCLGBM\"))\n",
    "        elif model_name == \"XGB\":\n",
    "            from xgboost import XGBClassifier\n",
    "            return XGBClassifier(**model_params(\"XGB\"))\n",
    "        elif model_name == \"RF\":\n",
    "            from sklearn.ensemble import RandomForestClassifier\n",
    "            return RandomForestClassifier(**model_params(\"RF\"))\n",
    "        elif model_name == \"CatBoost\":\n",
    "            from catboost import CatBoostClassifier\n",
    "            return CatBoostClassifier(**model_params(\"CatBoost\"))\n",
    "        elif model_name == \"SVM\":\n",
    "            from sklearn.svm import SVC\n",
    "            return SVC(**model_params(\"SVM\"))\n",
    "        else:\n",
    "            raise ValueError(\"Invalid model name.\")\n",
    "\n",
    "    model = get_model(model_name)\n",
    "\n",
    "    # 튜닝 방법에 따른 분기\n",
    "    if method == \"grid\":\n",
    "        search = GridSearchCV(model, param_dist, cv=3, scoring='accuracy', n_jobs=-1, verbose=1)\n",
    "    elif method == \"random\":\n",
    "        search = RandomizedSearchCV(model, param_dist, n_iter=n_iter, cv=3, scoring='accuracy', n_jobs=-1, verbose=1, random_state=42)\n",
    "    elif method == \"bayesian\":\n",
    "        search = BayesSearchCV(model, param_dist, n_iter=n_iter, cv=3, scoring='accuracy', n_jobs=-1, verbose=1, random_state=42)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid method. Choose from 'grid', 'random', or 'bayesian'.\")\n",
    "\n",
    "    # 하이퍼파라미터 튜닝 수행\n",
    "    search.fit(X_train, y_train)\n",
    "\n",
    "    print(f\"Best parameters for {model_name} using {method.capitalize()}SearchCV: {search.best_params_}\")\n",
    "    return search.best_estimator_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"real_final_df.csv\"\n",
    "model_name = \"LGBM\" # LGBM/XGB/RF/CatBoost/FCLGBM/SVM\n",
    "seve_name = \"savefile.csv\"\n",
    "split_type = \"random\" # random/time/randomcv\n",
    "drop_colunm = [\"target\", \"ID\", \"dir_prob_ts\", \"Unnamed 0\"]\n",
    "target_colunm = \"target\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from model.train import train_model, test, final_train_model\n",
    "from dataloader.dataset_load import data_split, _Dataset\n",
    "\n",
    "import csv\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = model_params(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파일 호출\n",
    "data_path: str = \"data\"\n",
    "df: pd.DataFrame = pd.read_csv(os.path.join(data_path, dataset_name))\n",
    "df.columns = df.columns.str.replace(r'[^\\w\\s]', '', regex=True)\n",
    "submission_df: pd.DataFrame = pd.read_csv(os.path.join(data_path, \"test.csv\")) # ID, target 열만 가진 데이터 미리 호출"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 타겟 변수를 제외한 변수를 forwardfill, -999로 결측치 대체\n",
    "_target = df[\"target\"]\n",
    "df = df.ffill().fillna(-999).assign(target = _target)\n",
    "\n",
    "# _type에 따라 train, test 분리\n",
    "train_df = df.loc[df[\"_type\"]==\"train\"].drop(columns=[\"_type\"])\n",
    "test_df = df.loc[df[\"_type\"]==\"test\"].drop(columns=[\"_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Best parameters for LGBM using BayesianSearchCV: OrderedDict([('boosting_type', 'gbdt'), ('learning_rate', 0.015732043600075817), ('metric', 'multi_logloss'), ('n_estimators', 52), ('num_class', 4), ('num_leaves', 97), ('objective', 'multiclass'), ('random_state', 42), ('verbose', 0)])\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = data_split(split_type, train_df, drop_colunm, target_colunm)\n",
    "\n",
    "best_model = hyperparameter_tuning(model_name, params, x_train, y_train, method=\"bayesian\", n_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LGBMClassifier(learning_rate=0.015732043600075817, metric=&#x27;multi_logloss&#x27;,\n",
       "               n_estimators=52, num_class=4, num_leaves=97,\n",
       "               objective=&#x27;multiclass&#x27;, random_state=42, verbose=0)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;LGBMClassifier<span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>LGBMClassifier(learning_rate=0.015732043600075817, metric=&#x27;multi_logloss&#x27;,\n",
       "               n_estimators=52, num_class=4, num_leaves=97,\n",
       "               objective=&#x27;multiclass&#x27;, random_state=42, verbose=0)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "LGBMClassifier(learning_rate=0.015732043600075817, metric='multi_logloss',\n",
       "               n_estimators=52, num_class=4, num_leaves=97,\n",
       "               objective='multiclass', random_state=42, verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ai_tech",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}