analysis/notebooks/model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d2713fa",
   "metadata": {},
   "source": [
    "# Coffee Data: Classification model training ☕☕☕\n",
    "\n",
    "🕵🏿‍♀️ To break the problem down we are going to look at a classification problem: predicting if a give coffee sample will have a `total_cup_points` of over 85 given the green and processing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e97e7b8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from data_prep import handle_na_values, split_data, convert_bag_weight, total_points_over_85\n",
    "from train import train_logistic_regression, train_decision_tree, train_random_forest, check_feature_importance\n",
    "from evaluate import validate_model, print_model_evaluation, model_evaluation\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "51a10221",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../data/merged_data_cleaned.csv\",  index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cc046d9-5d83-431d-b734-d75ab8057ec0",
   "metadata": {},
   "source": [
    "## Data preparation\n",
    "\n",
    "- make column names consistant\n",
    "- replace some na values and remove rows with other na values\n",
    "- convert bag weight\n",
    "- define numerical and categorical features\n",
    "- split data into train, validation, test sets and extract target value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f05f6b25",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_732/4262472041.py:1: FutureWarning: The default value of regex will change from True to False in a future version. In addition, single character regular expressions will *not* be treated as literal strings when regex=True.\n",
      "  df.columns = df.columns.str.lower().str.replace(\".\", \"_\")\n"
     ]
    }
   ],
   "source": [
    "df.columns = df.columns.str.lower().str.replace(\".\", \"_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1c3178bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = handle_na_values(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cc50e320-8e2f-424b-a770-0c70d147e94c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"bag_weight\"] = df[\"bag_weight\"].apply(lambda weight_str: convert_bag_weight(weight_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cfe578f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_features = [\n",
    "    \"moisture\",\n",
    "    \"category_one_defects\",\n",
    "    \"quakers\",\n",
    "    \"category_two_defects\",\n",
    "    \"altitude_mean_meters\",\n",
    "    \"bag_weight\"\n",
    "]\n",
    "\n",
    "categorical_features = [\n",
    "    \"color\",\n",
    "    \"species\",\n",
    "    \"owner\",\n",
    "    \"country_of_origin\",\n",
    "    \"farm_name\",\n",
    "    \"mill\",\n",
    "    \"company\",\n",
    "    \"region\",\n",
    "    \"producer\",\n",
    "    \"in_country_partner\",\n",
    "    \"harvest_year\",\n",
    "    \"owner_1\",\n",
    "    \"variety\",\n",
    "    \"processing_method\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9aecde63",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = numerical_features + categorical_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "81a7521f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of training set: 536, validation set: 179, test set: 179\n"
     ]
    }
   ],
   "source": [
    "df_train, df_val, df_test, y_train, y_val, y_test, df_full_train = split_data(df, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b842726c-34e2-4837-905b-bffe91a05c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set target y values to 1 if above 85 and 0 if 85 or lower total cup score\n",
    "y_train = total_points_over_85(y_train)\n",
    "y_val = total_points_over_85(y_val)\n",
    "y_test = total_points_over_85(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b54e549",
   "metadata": {},
   "source": [
    "## Model training\n",
    "\n",
    "- train three different models and evaluate a base line evaluation to which performs best with ROC AUC, RSME, Accuracy\n",
    "- see how each feature affects the base score for that model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2eb25dd2-d408-463c-a407-70a142760762",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_set = numerical_features + categorical_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2fdc30a",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b2eee359",
   "metadata": {},
   "outputs": [],
   "source": [
    "dv_lr, model_lr = train_logistic_regression(df_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9f8b6e9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94\n",
      "Roc Auc: 0.55\n",
      "Rsme: 0.24\n"
     ]
    }
   ],
   "source": [
    "y_pred_lr = validate_model(df_val, y_val, dv_lr, model_lr)\n",
    "\n",
    "print_model_evaluation(y_val, y_pred_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f8571220-3e2f-40bb-b4db-3b52db7bb0f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_removed</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>rsme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>harvest_year</td>\n",
       "      <td>0.955307</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.211407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>category_one_defects</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>quakers</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bag_weight</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>color</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>owner</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>country_of_origin</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>farm_name</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>region</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>producer</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>in_country_partner</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>owner_1</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>variety</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>moisture</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>category_two_defects</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>altitude_mean_meters</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>species</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mill</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>company</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>processing_method</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         feature_removed  accuracy       auc      rsme\n",
       "16          harvest_year  0.955307  0.555556  0.211407\n",
       "1   category_one_defects  0.949721  0.552614  0.224231\n",
       "2                quakers  0.949721  0.552614  0.224231\n",
       "5             bag_weight  0.949721  0.552614  0.224231\n",
       "6                  color  0.949721  0.552614  0.224231\n",
       "8                  owner  0.949721  0.552614  0.224231\n",
       "9      country_of_origin  0.949721  0.552614  0.224231\n",
       "10             farm_name  0.949721  0.552614  0.224231\n",
       "13                region  0.949721  0.552614  0.224231\n",
       "14              producer  0.949721  0.552614  0.224231\n",
       "15    in_country_partner  0.949721  0.552614  0.224231\n",
       "17               owner_1  0.949721  0.552614  0.224231\n",
       "18               variety  0.949721  0.552614  0.224231\n",
       "0               moisture  0.944134  0.549673  0.236360\n",
       "3   category_two_defects  0.944134  0.549673  0.236360\n",
       "4   altitude_mean_meters  0.944134  0.549673  0.236360\n",
       "7                species  0.944134  0.549673  0.236360\n",
       "11                  mill  0.944134  0.549673  0.236360\n",
       "12               company  0.944134  0.549673  0.236360\n",
       "19     processing_method  0.944134  0.549673  0.236360"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_df = check_feature_importance(feature_set, df_train, y_train, df_val, y_val, model_type=\"LR\")\n",
    "scores_df.sort_values(by=[\"accuracy\", \"auc\", \"rsme\"], ascending=[0, 0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d224c4ff",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c3022ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dv_dt, model_dt = train_decision_tree(df_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3ef46edf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95\n",
      "Roc Auc: 0.50\n",
      "Rsme: 0.22\n"
     ]
    }
   ],
   "source": [
    "y_pred_dt = validate_model(df_val, y_val, dv_dt, model_dt)\n",
    "\n",
    "print_model_evaluation(y_val, y_pred_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "25d866a5-1135-4b77-af34-a3796ef0b985",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_removed</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>rsme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>moisture</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>category_one_defects</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>quakers</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>category_two_defects</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>altitude_mean_meters</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bag_weight</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>color</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>species</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>owner</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>country_of_origin</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>farm_name</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mill</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>company</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>region</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>producer</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>in_country_partner</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>harvest_year</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>owner_1</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>variety</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>processing_method</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         feature_removed  accuracy  auc      rsme\n",
       "0               moisture  0.949721  0.5  0.224231\n",
       "1   category_one_defects  0.949721  0.5  0.224231\n",
       "2                quakers  0.949721  0.5  0.224231\n",
       "3   category_two_defects  0.949721  0.5  0.224231\n",
       "4   altitude_mean_meters  0.949721  0.5  0.224231\n",
       "5             bag_weight  0.949721  0.5  0.224231\n",
       "6                  color  0.949721  0.5  0.224231\n",
       "7                species  0.949721  0.5  0.224231\n",
       "8                  owner  0.949721  0.5  0.224231\n",
       "9      country_of_origin  0.949721  0.5  0.224231\n",
       "10             farm_name  0.949721  0.5  0.224231\n",
       "11                  mill  0.949721  0.5  0.224231\n",
       "12               company  0.949721  0.5  0.224231\n",
       "13                region  0.949721  0.5  0.224231\n",
       "14              producer  0.949721  0.5  0.224231\n",
       "15    in_country_partner  0.949721  0.5  0.224231\n",
       "16          harvest_year  0.949721  0.5  0.224231\n",
       "17               owner_1  0.949721  0.5  0.224231\n",
       "18               variety  0.949721  0.5  0.224231\n",
       "19     processing_method  0.949721  0.5  0.224231"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_df = check_feature_importance(feature_set, df_train, y_train, df_val, y_val, model_type=\"DT\")\n",
    "scores_df.sort_values(by=[\"accuracy\", \"auc\", \"rsme\"], ascending=[0, 0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92e36cd4",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "87ffda5f-3cf6-4645-be8e-efc14d89ea17",
   "metadata": {},
   "outputs": [],
   "source": [
    "dv_rf, model_rf = train_random_forest(df_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8bafd716",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95\n",
      "Roc Auc: 0.50\n",
      "Rsme: 0.22\n"
     ]
    }
   ],
   "source": [
    "y_pred_rf = validate_model(df_val, y_val, dv_dt, model_dt)\n",
    "\n",
    "print_model_evaluation(y_val, y_pred_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "056ff7b8-f561-4216-b335-a7478edf21a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_removed</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>rsme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>category_one_defects</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>owner</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>owner_1</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>in_country_partner</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>moisture</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>category_two_defects</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>altitude_mean_meters</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bag_weight</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>country_of_origin</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>region</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>producer</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>harvest_year</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>variety</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.497059</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>quakers</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>color</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>species</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>farm_name</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mill</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>company</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>processing_method</td>\n",
       "      <td>0.938547</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.247896</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         feature_removed  accuracy       auc      rsme\n",
       "1   category_one_defects  0.949721  0.552614  0.224231\n",
       "8                  owner  0.949721  0.552614  0.224231\n",
       "17               owner_1  0.949721  0.552614  0.224231\n",
       "15    in_country_partner  0.944134  0.549673  0.236360\n",
       "0               moisture  0.944134  0.497059  0.236360\n",
       "3   category_two_defects  0.944134  0.497059  0.236360\n",
       "4   altitude_mean_meters  0.944134  0.497059  0.236360\n",
       "5             bag_weight  0.944134  0.497059  0.236360\n",
       "9      country_of_origin  0.944134  0.497059  0.236360\n",
       "13                region  0.944134  0.497059  0.236360\n",
       "14              producer  0.944134  0.497059  0.236360\n",
       "16          harvest_year  0.944134  0.497059  0.236360\n",
       "18               variety  0.944134  0.497059  0.236360\n",
       "2                quakers  0.938547  0.494118  0.247896\n",
       "6                  color  0.938547  0.494118  0.247896\n",
       "7                species  0.938547  0.494118  0.247896\n",
       "10             farm_name  0.938547  0.494118  0.247896\n",
       "11                  mill  0.938547  0.494118  0.247896\n",
       "12               company  0.938547  0.494118  0.247896\n",
       "19     processing_method  0.938547  0.494118  0.247896"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_df = check_feature_importance(feature_set, df_train, y_train, df_val, y_val, model_type=\"RF\")\n",
    "scores_df.sort_values(by=[\"accuracy\", \"auc\", \"rsme\"], ascending=[0, 0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be604aa9-513d-4d0e-9728-fb02fb116456",
   "metadata": {},
   "source": [
    "The model and selection of features that performed the best was the Logistic Regression with the Harvest year feature removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e059c023-a795-4965-a745-5527198b01e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_subset = [\n",
    "    \"moisture\",\n",
    "     \"category_one_defects\",\n",
    "     \"quakers\",\n",
    "     \"category_two_defects\",\n",
    "     \"altitude_mean_meters\",\n",
    "     \"bag_weight\",\n",
    "     \"color\",\n",
    "     \"species\",\n",
    "     \"owner\",\n",
    "     \"country_of_origin\",\n",
    "     \"farm_name\",\n",
    "     \"mill\",\n",
    "     \"company\",\n",
    "     \"region\",\n",
    "     \"producer\",\n",
    "     \"in_country_partner\",\n",
    "     \"owner_1\",\n",
    "     \"variety\",\n",
    "     \"processing_method\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "df05bd21-6a15-4b87-ab5b-ac634df69c19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96\n",
      "Roc Auc: 0.56\n",
      "Rsme: 0.21\n"
     ]
    }
   ],
   "source": [
    "dv_lr2, model_lr2 = train_logistic_regression(df_train[features_subset], y_train)\n",
    "y_pred_lr2 = validate_model(df_val, y_val, dv_lr2, model_lr2)\n",
    "\n",
    "print_model_evaluation(y_val, y_pred_lr2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "184c8a36",
   "metadata": {},
   "source": [
    "## Parameter Tuning\n",
    "\n",
    "Taking the model we decided upon in the previous step we now look to see which parameters for that model can be tuned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "edcc5961-72a3-42b3-af2b-196406e253e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import warnings\n",
    "import os\n",
    "\n",
    "if not sys.warnoptions:\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    os.environ[\"PYTHONWARNINGS\"] = \"ignore\" # Also affect subprocesses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5d80c6a0-0da8-445c-92dd-222e29755fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dv_lr, model_lr = train_logistic_regression(df_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "73cd8284",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>random_state</th>\n",
       "      <th>c_value</th>\n",
       "      <th>max_iter</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>rsme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>100</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>10</td>\n",
       "      <td>1.00</td>\n",
       "      <td>100</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>32</td>\n",
       "      <td>1.00</td>\n",
       "      <td>100</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>42</td>\n",
       "      <td>1.00</td>\n",
       "      <td>100</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.552614</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0.01</td>\n",
       "      <td>100</td>\n",
       "      <td>0.949721</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.224231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>42</td>\n",
       "      <td>1.00</td>\n",
       "      <td>10000</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>42</td>\n",
       "      <td>10.00</td>\n",
       "      <td>100</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>42</td>\n",
       "      <td>10.00</td>\n",
       "      <td>500</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>42</td>\n",
       "      <td>10.00</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>42</td>\n",
       "      <td>10.00</td>\n",
       "      <td>10000</td>\n",
       "      <td>0.944134</td>\n",
       "      <td>0.549673</td>\n",
       "      <td>0.236360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    random_state  c_value  max_iter  accuracy       auc      rsme\n",
       "8              2     1.00       100  0.949721  0.552614  0.224231\n",
       "24            10     1.00       100  0.949721  0.552614  0.224231\n",
       "40            32     1.00       100  0.949721  0.552614  0.224231\n",
       "56            42     1.00       100  0.949721  0.552614  0.224231\n",
       "0              2     0.01       100  0.949721  0.500000  0.224231\n",
       "..           ...      ...       ...       ...       ...       ...\n",
       "59            42     1.00     10000  0.944134  0.549673  0.236360\n",
       "60            42    10.00       100  0.944134  0.549673  0.236360\n",
       "61            42    10.00       500  0.944134  0.549673  0.236360\n",
       "62            42    10.00      1000  0.944134  0.549673  0.236360\n",
       "63            42    10.00     10000  0.944134  0.549673  0.236360\n",
       "\n",
       "[64 rows x 6 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = []\n",
    "\n",
    "for random_state in [2, 10, 32, 42]:\n",
    "    for c in [0.01, 0.1, 1, 10]:\n",
    "        for no_iter in [100, 500, 1000, 10000]:\n",
    "            dv_lr, model_lr = train_logistic_regression(df_train, y_train, c=c, max_iter=no_iter, random_state=random_state)\n",
    "\n",
    "            y_pred_lr = validate_model(df_val, y_val, dv_lr, model_lr)\n",
    "\n",
    "            scores.append((random_state, c, no_iter, *model_evaluation(y_val, y_pred_lr)))\n",
    "\n",
    "cols = [\"random_state\", \"c_value\", \"max_iter\", \"accuracy\", \"auc\", \"rsme\"]\n",
    "\n",
    "parameter_tuning_scores_df = pd.DataFrame(scores, columns=cols)\n",
    "parameter_tuning_scores_df.sort_values(by=[\"accuracy\", \"auc\", \"rsme\"], ascending=[0, 0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d918871-c5b4-45ac-a23a-d7f41fc065c6",
   "metadata": {},
   "source": [
    "## Check model with test data\n",
    "\n",
    "Now we have our model `Logistic Regression`and final parameters `c=1, max_iter=100, random_state=2`we train the final model with our full training set and validate it against the test data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "00d7802d-c1d8-484a-a34e-280d7a1c2e3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94\n",
      "Roc Auc: 0.49\n",
      "Rsme: 0.25\n"
     ]
    }
   ],
   "source": [
    "dv_final, model_final = train_logistic_regression(df_full_train[features_subset], total_points_over_85(df_full_train[\"total_cup_points\"].values),c=1, max_iter=100, random_state=2)\n",
    "\n",
    "y_pred_final = validate_model(df_test, y_test, dv_final, model_final)\n",
    "\n",
    "print_model_evaluation(y_val, y_pred_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0be49c4-6dcd-4392-ba11-748b07780aea",
   "metadata": {},
   "source": [
    "This model is compareable with the train set tested against the validation set though the performance has been reduced slightly which suggests some overfitting. A gridsearch looking at different folds (splits) of the training and test set might reveal more."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40f532db-063d-44f5-9206-dd438348a1b2",
   "metadata": {},
   "source": [
    "## Using the model\n",
    "\n",
    "Let's test the model with a single input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3361a552-c0e2-45cc-8514-30b43b57fe41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "moisture                                        0.11\n",
       "category_one_defects                               0\n",
       "quakers                                          0.0\n",
       "category_two_defects                               2\n",
       "altitude_mean_meters                          1350.0\n",
       "bag_weight                                         1\n",
       "color                                          Green\n",
       "species                                      Arabica\n",
       "owner                                        cadexsa\n",
       "country_of_origin                           Honduras\n",
       "farm_name                                     bethel\n",
       "mill                                         cadexsa\n",
       "company                                      cadexsa\n",
       "region                                       marcala\n",
       "producer                                 Omar Acosta\n",
       "in_country_partner      Instituto Hondureño del Café\n",
       "owner_1                                      CADEXSA\n",
       "variety                                       Catuai\n",
       "processing_method                       Washed / Wet\n",
       "Name: 739, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = df_full_train[features_subset].iloc[1]\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "122bc8a3-6d34-4723-af47-f6e92911dbb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val = dv_final.transform([sample])\n",
    "y_pred = model_final.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "591fe6fc-b59e-436f-9264-75120c3f1e4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample predicted to score over 85 points: False\n"
     ]
    }
   ],
   "source": [
    "print(f\"sample predicted to score over 85 points: {str(y_pred[0] == 1)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}