TF model (1 lag).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building and fitting NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as sparse\n",
    "from scipy.sparse.linalg import spsolve\n",
    "import random\n",
    "import os\n",
    "import scipy.stats as ss\n",
    "import scipy\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from catboost import CatBoostClassifier, Pool, sum_models, to_classifier\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "import implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = pd.read_csv(\"../input/feature-creating-v2/train_3lags_v3.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df.loc[df['order_count'] == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#threshold = 0.0005\n",
    "#counts = df['service_title'].value_counts(normalize=True)\n",
    "#df2 = df.loc[df['service_title'].isin(counts[counts > threshold].index), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub15 = pd.read_csv(\"../input/catboost-fitting-smart/submission_15.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#d_134 = df2.loc[df2['service_title'] == 134][:121500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#d_98 = df2.loc[df2['service_title'] == 98][:121500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = df2.loc[df2['service_title'] != 134]\n",
    "#df2 = df2.loc[df2['service_title'] != 98]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = pd.concat([df2,d_134], axis=0)\n",
    "#df2 = pd.concat([df2,d_98], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2.to_csv('train_3lags_semibalanced.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Downloading prepared data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(\"../input/irkutsk/train_3lags_semibalanced.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = pd.read_csv(\"../input/irkutsk/train_3lags_v4.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2.drop(['Unnamed: 0'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98      0.082696\n",
       "134     0.082696\n",
       "4       0.079054\n",
       "603     0.051325\n",
       "604     0.039309\n",
       "1020    0.035067\n",
       "651     0.034391\n",
       "949     0.032985\n",
       "207     0.031835\n",
       "1259    0.028046\n",
       "1169    0.025519\n",
       "907     0.023750\n",
       "234     0.020259\n",
       "1205    0.017263\n",
       "1220    0.016864\n",
       "524     0.015905\n",
       "901     0.015231\n",
       "412     0.013853\n",
       "805     0.013103\n",
       "155     0.012644\n",
       "826     0.012597\n",
       "870     0.011420\n",
       "218     0.010956\n",
       "966     0.009862\n",
       "692     0.009389\n",
       "569     0.009015\n",
       "1107    0.008706\n",
       "178     0.008497\n",
       "573     0.007997\n",
       "491     0.007371\n",
       "Name: service_title, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2['service_title'].value_counts(normalize=True)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub = pd.read_csv(\"../input/nn-sub/nn_sub_5fold_2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2['service_title'].value_counts(normalize=True)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub['service_title'] = 1259"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub.to_csv('check_1259.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub['service_title'].value_counts(normalize=True)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df2[['service_title', 'order_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.drop(['service',\n",
    "'service_title',\n",
    "'mfc',\n",
    "'internal_status',\n",
    "'external_status',\n",
    "'order_type',\n",
    "'department_id',\n",
    "'custom_service_id',\n",
    "'service_level',\n",
    "'is_subdep',\n",
    "'is_csid',\n",
    "'proc_time',\n",
    "'dayofweek',\n",
    "'day_part',\n",
    "'person',\n",
    "'sole',\n",
    "'legal',\n",
    "'auto_ping_queue',\n",
    "'win_count',\n",
    "'month',\n",
    "'week',\n",
    "'year'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df2.loc[df2['order_count'] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_test.loc[df_test['order_count'] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.reset_index(inplace=True)\n",
    "df_test.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.drop(['index'], axis=1, inplace=True)\n",
    "df_test.drop(['index'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mask = (df_train_mask['order_count'] == 5)\n",
    "#z_valid = df_train_mask[mask]\n",
    "\n",
    "#df_train_mask.loc[mask, 'service_4'] = np.nan\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['requester', 'service_1', 'service_title_1', 'mfc_1',\n",
       "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
       "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
       "       'is_subdep_1', 'is_csid_1', 'proc_time_1', 'dayofweek_1', 'day_part_1',\n",
       "       'month_1', 'week_1', 'year_1', 'person_1', 'sole_1', 'legal_1',\n",
       "       'auto_ping_queue_1', 'win_count_1', 'service_2', 'service_title_2',\n",
       "       'mfc_2', 'internal_status_2', 'external_status_2', 'order_type_2',\n",
       "       'department_id_2', 'custom_service_id_2', 'service_level_2',\n",
       "       'is_subdep_2', 'is_csid_2', 'proc_time_2', 'dayofweek_2', 'day_part_2',\n",
       "       'month_2', 'week_2', 'year_2', 'person_2', 'sole_2', 'legal_2',\n",
       "       'auto_ping_queue_2', 'win_count_2', 'service_3', 'service_title_3',\n",
       "       'mfc_3', 'internal_status_3', 'external_status_3', 'order_type_3',\n",
       "       'department_id_3', 'custom_service_id_3', 'service_level_3',\n",
       "       'is_subdep_3', 'is_csid_3', 'proc_time_3', 'dayofweek_3', 'day_part_3',\n",
       "       'month_3', 'week_3', 'year_3', 'person_3', 'sole_3', 'legal_3',\n",
       "       'auto_ping_queue_3', 'win_count_3', 'requester_type', 'gender', 'age',\n",
       "       'order_count'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2','service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'order_type_3', 'department_id_3', 'custom_service_id_3',\n",
    "       'service_level_3', 'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', 'day_part_3', 'person_3', 'sole_3', 'legal_3',\n",
    "       'auto_ping_queue_3',\n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'department_id_2', 'month_2', 'week_2', 'year_2',\n",
    "       'custom_service_id_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'department_id_3', 'custom_service_id_3',\n",
    "       'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train.drop(['requester'], axis=1)\n",
    "y = df_test['service_title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.drop(['service_2',\n",
    "'service_title_2',\n",
    "'mfc_2',\n",
    "'internal_status_2',\n",
    "'external_status_2',\n",
    "'order_type_2',\n",
    "'department_id_2',\n",
    "'custom_service_id_2',\n",
    "'service_level_2',\n",
    "'is_subdep_2',\n",
    "'is_csid_2',\n",
    "'proc_time_2',\n",
    "'dayofweek_2',\n",
    "'day_part_2',\n",
    "'person_2',\n",
    "'sole_2',\n",
    "'legal_2',\n",
    "'auto_ping_queue_2',\n",
    "'win_count_2',\n",
    "'month_2',\n",
    "'week_2',\n",
    "'year_2'], axis=1 , inplace=True)\n",
    "\n",
    "X.drop(['service_3',\n",
    "'service_title_3',\n",
    "'mfc_3',\n",
    "'internal_status_3',\n",
    "'external_status_3',\n",
    "'order_type_3',\n",
    "'department_id_3',\n",
    "'custom_service_id_3',\n",
    "'service_level_3',\n",
    "'is_subdep_3',\n",
    "'is_csid_3',\n",
    "'proc_time_3',\n",
    "'dayofweek_3',\n",
    "'day_part_3',\n",
    "'person_3',\n",
    "'sole_3',\n",
    "'legal_3',\n",
    "'auto_ping_queue_3',\n",
    "'win_count_3',\n",
    "'month_3',\n",
    "'week_3',\n",
    "'year_3'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'requester_type', 'gender']\n",
    "\n",
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[cat] = X[cat].astype('Int64')\n",
    "X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X[cat] = X[cat].astype('Int64')\n",
    "#X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[categorical] = X[categorical].fillna('NA')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to reduce ram usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df):\n",
    "    \"\"\" \n",
    "    iterate through all the columns of a dataframe and \n",
    "    modify the data type to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage of dataframe is {:.2f}' \n",
    "                     'MB').format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max <\\\n",
    "                  np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max <\\\n",
    "                   np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max <\\\n",
    "                   np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max <\\\n",
    "                   np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max <\\\n",
    "                   np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max <\\\n",
    "                   np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage after optimization is: {:.2f}' \n",
    "                              'MB').format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) \n",
    "                                             / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 252.21MB\n",
      "Memory usage after optimization is: 75.66MB\n",
      "Decreased by 70.0%\n"
     ]
    }
   ],
   "source": [
    "X = reduce_mem_usage(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X[categorical]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['person_1'] = X['person_1'].astype('int32')\n",
    "X['sole_1'] = X['sole_1'].astype('int32')\n",
    "X['legal_1'] = X['legal_1'].astype('int32')\n",
    "X['auto_ping_queue_1'] = X['auto_ping_queue_1'].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "labeling = []\n",
    "for col in X.columns:\n",
    "    d = pd.DataFrame()\n",
    "    d[col] = X[col].unique()\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.fit(d[col])\n",
    "    d[col+'_l'] = le.transform(d[col])\n",
    "    d = d.sort_values(by=[col+'_l']).reset_index(drop=True)\n",
    "    labeling.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame()\n",
    "d['service_title'] = y.unique()\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(d['service_title'])\n",
    "d['service_title'+'_l'] = le.transform(d['service_title'])\n",
    "d = d.sort_values(by=['service_title'+'_l']).reset_index(drop=True)\n",
    "labeling_y = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>service_1</th>\n",
       "      <th>service_1_l</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2266851</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2266858</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2266865</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2266872</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46880280</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>62311942</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64279142</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>64279302</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64279389</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>64279525</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64279644</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>64279761</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>65129676</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>81603166</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>85744749</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>97794844</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>113491165</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>113491183</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>113491201</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>113491284</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    service_1  service_1_l\n",
       "0          43            0\n",
       "1     2266851            1\n",
       "2     2266858            2\n",
       "3     2266865            3\n",
       "4     2266872            4\n",
       "5    46880280            5\n",
       "6    62311942            6\n",
       "7    64279142            7\n",
       "8    64279302            8\n",
       "9    64279389            9\n",
       "10   64279525           10\n",
       "11   64279644           11\n",
       "12   64279761           12\n",
       "13   65129676           13\n",
       "14   81603166           14\n",
       "15   85744749           15\n",
       "16   97794844           16\n",
       "17  113491165           17\n",
       "18  113491183           18\n",
       "19  113491201           19\n",
       "20  113491284           20"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labeling[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in X.columns:\n",
    "    X[col] = X[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>service_1</th>\n",
       "      <th>service_title_1</th>\n",
       "      <th>mfc_1</th>\n",
       "      <th>internal_status_1</th>\n",
       "      <th>external_status_1</th>\n",
       "      <th>order_type_1</th>\n",
       "      <th>department_id_1</th>\n",
       "      <th>custom_service_id_1</th>\n",
       "      <th>service_level_1</th>\n",
       "      <th>is_subdep_1</th>\n",
       "      <th>...</th>\n",
       "      <th>day_part_1</th>\n",
       "      <th>month_1</th>\n",
       "      <th>week_1</th>\n",
       "      <th>year_1</th>\n",
       "      <th>person_1</th>\n",
       "      <th>sole_1</th>\n",
       "      <th>legal_1</th>\n",
       "      <th>auto_ping_queue_1</th>\n",
       "      <th>requester_type</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>493</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>72</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>110</td>\n",
       "      <td>89</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>1026</td>\n",
       "      <td>124</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>671</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>533</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469236</th>\n",
       "      <td>13</td>\n",
       "      <td>946</td>\n",
       "      <td>38</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469237</th>\n",
       "      <td>0</td>\n",
       "      <td>790</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469238</th>\n",
       "      <td>0</td>\n",
       "      <td>193</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>72</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469239</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>7</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469240</th>\n",
       "      <td>0</td>\n",
       "      <td>82</td>\n",
       "      <td>130</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1469241 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         service_1  service_title_1  mfc_1  internal_status_1  \\\n",
       "0                0              493      0                  7   \n",
       "1                0              110     89                  7   \n",
       "2                4             1026    124                  7   \n",
       "3                0              671      1                  7   \n",
       "4                0              533      2                  8   \n",
       "...            ...              ...    ...                ...   \n",
       "1469236         13              946     38                  7   \n",
       "1469237          0              790     17                  7   \n",
       "1469238          0              193     36                  7   \n",
       "1469239          0                2     38                  7   \n",
       "1469240          0               82    130                  7   \n",
       "\n",
       "         external_status_1 order_type_1  department_id_1  custom_service_id_1  \\\n",
       "0                       21            3               72                   25   \n",
       "1                       21            3                8                   28   \n",
       "2                        4            3               15                    0   \n",
       "3                        4            0               72                   26   \n",
       "4                        4            3               16                   18   \n",
       "...                    ...          ...              ...                  ...   \n",
       "1469236                 21            3               72                    0   \n",
       "1469237                 21            3               16                   20   \n",
       "1469238                 21            3               72                   17   \n",
       "1469239                 21            3               31                   29   \n",
       "1469240                  4            0                8                   28   \n",
       "\n",
       "        service_level_1  is_subdep_1  ...  day_part_1  month_1 week_1  year_1  \\\n",
       "0                     0            0  ...           1        3     14       0   \n",
       "1                     0            0  ...           1        6     28       0   \n",
       "2                     0            0  ...           1        7     33       0   \n",
       "3                     0            0  ...           1       11     51       0   \n",
       "4                     4            0  ...           1        1      6       0   \n",
       "...                 ...          ...  ...         ...      ...    ...     ...   \n",
       "1469236               0            0  ...           1        0      2       1   \n",
       "1469237               4            0  ...           1       10     46       0   \n",
       "1469238               0            0  ...           1        1      7       0   \n",
       "1469239               0            0  ...           2        0      2       0   \n",
       "1469240               0            0  ...           1        0      3       1   \n",
       "\n",
       "         person_1  sole_1  legal_1  auto_ping_queue_1  requester_type  gender  \n",
       "0               0       0        0                  1               0       0  \n",
       "1               0       0        0                  1               0       1  \n",
       "2               1       0        0                  1               0       1  \n",
       "3               0       0        0                  1               0       0  \n",
       "4               0       0        0                  1               0       1  \n",
       "...           ...     ...      ...                ...             ...     ...  \n",
       "1469236         1       0        0                  1               0       1  \n",
       "1469237         0       0        0                  1               0       1  \n",
       "1469238         0       0        0                  1               0       1  \n",
       "1469239         0       0        0                  1               0       1  \n",
       "1469240         0       0        0                  1               0       1  \n",
       "\n",
       "[1469241 rows x 22 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.map(labeling_y.set_index('service_title').to_dict()['service_title'+'_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "#s_w = compute_class_weight(class_weight='balanced', classes=labeling_y['service_title_l'], y=y)\n",
    "\n",
    "weights_l = labeling_y[:]\n",
    "weights_l['weights'] = 1\n",
    "\n",
    "weights = pd.DataFrame(y)\n",
    "weights = pd.merge(weights, weights_l, how='left', left_on='service_title', right_on='service_title_l')\n",
    "#weights = np.array(weights['weights'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98      0.082696\n",
       "134     0.082696\n",
       "4       0.079054\n",
       "603     0.051325\n",
       "604     0.039309\n",
       "1020    0.035067\n",
       "651     0.034391\n",
       "949     0.032985\n",
       "207     0.031835\n",
       "1259    0.028046\n",
       "1169    0.025519\n",
       "907     0.023750\n",
       "234     0.020259\n",
       "1205    0.017263\n",
       "1220    0.016864\n",
       "524     0.015905\n",
       "901     0.015231\n",
       "412     0.013853\n",
       "805     0.013103\n",
       "155     0.012644\n",
       "826     0.012597\n",
       "870     0.011420\n",
       "218     0.010956\n",
       "966     0.009862\n",
       "692     0.009389\n",
       "569     0.009015\n",
       "1107    0.008706\n",
       "178     0.008497\n",
       "573     0.007997\n",
       "491     0.007371\n",
       "Name: service_title_y, dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights['service_title_y'].value_counts(normalize=True)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "weights['weights'].loc[weights['service_title_y'] == 4] = 0.4\n",
    "weights['weights'].loc[weights['service_title_y'] == 603] = 0.6\n",
    "weights['weights'].loc[weights['service_title_y'] == 98] = 0.9\n",
    "weights['weights'].loc[weights['service_title_y'] == 134] = 0.87"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = np.array(weights['weights'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "service_1 21\n",
      "service_title_1 1068\n",
      "mfc_1 212\n",
      "internal_status_1 9\n",
      "external_status_1 28\n",
      "order_type_1 4\n",
      "department_id_1 92\n",
      "custom_service_id_1 51\n",
      "service_level_1 5\n",
      "is_subdep_1 2\n",
      "is_csid_1 2\n",
      "dayofweek_1 7\n",
      "day_part_1 4\n",
      "month_1 12\n",
      "week_1 52\n",
      "year_1 2\n",
      "person_1 2\n",
      "sole_1 2\n",
      "legal_1 2\n",
      "auto_ping_queue_1 2\n",
      "requester_type 3\n",
      "gender 2\n"
     ]
    }
   ],
   "source": [
    "for col in X.columns:\n",
    "    print(col, len(X[col].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on TPU  grpc://10.0.0.2:8470\n",
      "REPLICAS:  8\n"
     ]
    }
   ],
   "source": [
    "# Detect hardware, return appropriate distribution strategy\n",
    "try:\n",
    "    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n",
    "    # set: this is always the case on Kaggle.\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    tpu = None\n",
    "\n",
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "BATCH_SIZE = 64 * strategy.num_replicas_in_sync\n",
    "LEARNING_RATE = 1e-3 * strategy.num_replicas_in_sync \n",
    "EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
       "       'external_status_1', 'order_type_1', 'department_id_1',\n",
       "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
       "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
       "       'sole_1', 'legal_1', 'auto_ping_queue_1', 'requester_type', 'gender'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X['service_1'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_dataset(idxT, idxV):\n",
    "    \n",
    "    trn_weights = weights[idxT, ]\n",
    "    val_weights = weights[idxV, ]\n",
    "    #n samples\n",
    "    \n",
    "    trn_input_ids = np.array(X.index)[idxT,]\n",
    "    \n",
    "    # Trainset\n",
    "    trn_input_service_1 = np.array(X['service_1'])[idxT,]\n",
    "    trn_input_service_title_1 = np.array(X['service_title_1'])[idxT,]\n",
    "    trn_input_mfc_1 = np.array(X['mfc_1'])[idxT,]\n",
    "    trn_input_internal_status_1 = np.array(X['internal_status_1'])[idxT,]\n",
    "    trn_input_external_status_1 = np.array(X['external_status_1'])[idxT,]\n",
    "    trn_input_order_type_1 = np.array(X['order_type_1'])[idxT,]\n",
    "    trn_input_department_id_1 = np.array(X['department_id_1'])[idxT,]\n",
    "    trn_input_custom_service_id_1 = np.array(X['custom_service_id_1'])[idxT,]\n",
    "    trn_input_service_level_1 = np.array(X['service_level_1'])[idxT,]\n",
    "    trn_input_is_subdep_1 = np.array(X['is_subdep_1'])[idxT,]\n",
    "    trn_input_is_csid_1 = np.array(X['is_csid_1'])[idxT,]\n",
    "    trn_input_dayofweek_1 = np.array(X['dayofweek_1'])[idxT,]\n",
    "    trn_input_day_part_1 = np.array(X['day_part_1'])[idxT,]\n",
    "    trn_input_month_1 = np.array(X['month_1'])[idxT,]\n",
    "    trn_input_week_1 = np.array(X['week_1'])[idxT,]\n",
    "    trn_input_year_1 = np.array(X['year_1'])[idxT,]\n",
    "    trn_input_person_1 = np.array(X['person_1'])[idxT,]\n",
    "    trn_input_sole_1 = np.array(X['sole_1'])[idxT,]\n",
    "    trn_input_legal_1 = np.array(X['legal_1'])[idxT,]\n",
    "    trn_input_auto_ping_queue_1 = np.array(X['auto_ping_queue_1'])[idxT,]\n",
    "    trn_input_requester_type = np.array(X['requester_type'])[idxT,]\n",
    "    trn_input_gender = np.array(X['gender'])[idxT,]\n",
    "    \n",
    "    \n",
    "    trn_service_title = np.array(pd.get_dummies(y))[idxT,].astype('int32')\n",
    "    \n",
    "    # Validation set\n",
    "    val_input_service_1 = np.array(X['service_1'])[idxV,]\n",
    "    val_input_service_title_1 = np.array(X['service_title_1'])[idxV,]\n",
    "    val_input_mfc_1 = np.array(X['mfc_1'])[idxV,]\n",
    "    val_input_internal_status_1 = np.array(X['internal_status_1'])[idxV,]\n",
    "    val_input_external_status_1 = np.array(X['external_status_1'])[idxV,]\n",
    "    val_input_order_type_1 = np.array(X['order_type_1'])[idxV,]\n",
    "    val_input_department_id_1 = np.array(X['department_id_1'])[idxV,]\n",
    "    val_input_custom_service_id_1 = np.array(X['custom_service_id_1'])[idxV,]\n",
    "    val_input_service_level_1 = np.array(X['service_level_1'])[idxV,]\n",
    "    val_input_is_subdep_1 = np.array(X['is_subdep_1'])[idxV,]\n",
    "    val_input_is_csid_1 = np.array(X['is_csid_1'])[idxV,]\n",
    "    val_input_dayofweek_1 = np.array(X['dayofweek_1'])[idxV,]\n",
    "    val_input_day_part_1 = np.array(X['day_part_1'])[idxV,]\n",
    "    val_input_month_1 = np.array(X['month_1'])[idxV,]\n",
    "    val_input_week_1 = np.array(X['week_1'])[idxV,]\n",
    "    val_input_year_1 = np.array(X['year_1'])[idxV,]\n",
    "    val_input_person_1 = np.array(X['person_1'])[idxV,]\n",
    "    val_input_sole_1 = np.array(X['sole_1'])[idxV,]\n",
    "    val_input_legal_1 = np.array(X['legal_1'])[idxV,]\n",
    "    val_input_auto_ping_queue_1 = np.array(X['auto_ping_queue_1'])[idxV,]\n",
    "    val_input_requester_type = np.array(X['requester_type'])[idxV,]\n",
    "    val_input_gender = np.array(X['gender'])[idxV,]\n",
    "    \n",
    "    \n",
    "    val_service_title = np.array(pd.get_dummies(y))[idxV,].astype('int32')\n",
    "    \n",
    "    # Generating tf.data object\n",
    "    train_dataset = (\n",
    "        tf.data.Dataset\n",
    "        .from_tensor_slices(({'service_1':trn_input_service_1, \n",
    "                              'service_title_1': trn_input_service_title_1,\n",
    "                              'mfc_1': trn_input_mfc_1,\n",
    "                              'internal_status_1': trn_input_internal_status_1,\n",
    "                              'external_status_1': trn_input_external_status_1,\n",
    "                              'order_type_1': trn_input_order_type_1,\n",
    "                              'department_id_1': trn_input_department_id_1,\n",
    "                              'custom_service_id_1': trn_input_custom_service_id_1,\n",
    "                              'service_level_1': trn_input_service_level_1,\n",
    "                              'is_subdep_1': trn_input_is_subdep_1,\n",
    "                              'is_csid_1': trn_input_is_csid_1,\n",
    "                              'dayofweek_1': trn_input_dayofweek_1,\n",
    "                              'day_part_1': trn_input_day_part_1,\n",
    "                              'month_1': trn_input_month_1,\n",
    "                              'week_1': trn_input_week_1,\n",
    "                              'year_1': trn_input_year_1,\n",
    "                              'person_1': trn_input_person_1,\n",
    "                              'sole_1': trn_input_sole_1,\n",
    "                              'legal_1': trn_input_legal_1,\n",
    "                              'auto_ping_queue_1': trn_input_auto_ping_queue_1,\n",
    "                              'requester_type': trn_input_requester_type,\n",
    "                              'gender': trn_input_gender}, \n",
    "                             {'service_title': trn_service_title}, trn_weights))\n",
    "        .shuffle(2048)\n",
    "        .batch(BATCH_SIZE)\n",
    "        .prefetch(AUTO)\n",
    "    )\n",
    "    \n",
    "    valid_dataset = (\n",
    "        tf.data.Dataset\n",
    "        .from_tensor_slices(({'service_1':val_input_service_1, \n",
    "                              'service_title_1': val_input_service_title_1,\n",
    "                              'mfc_1': val_input_mfc_1,\n",
    "                              'internal_status_1': val_input_internal_status_1,\n",
    "                              'external_status_1': val_input_external_status_1,\n",
    "                              'order_type_1': val_input_order_type_1,\n",
    "                              'department_id_1': val_input_department_id_1,\n",
    "                              'custom_service_id_1': val_input_custom_service_id_1,\n",
    "                              'service_level_1': val_input_service_level_1,\n",
    "                              'is_subdep_1': val_input_is_subdep_1,\n",
    "                              'is_csid_1': val_input_is_csid_1,\n",
    "                              'dayofweek_1': val_input_dayofweek_1,\n",
    "                              'day_part_1': val_input_day_part_1,\n",
    "                              'month_1': val_input_month_1,\n",
    "                              'week_1': val_input_week_1,\n",
    "                              'year_1': val_input_year_1,\n",
    "                              'person_1': val_input_person_1,\n",
    "                              'sole_1': val_input_sole_1,\n",
    "                              'legal_1': val_input_legal_1,\n",
    "                              'auto_ping_queue_1': val_input_auto_ping_queue_1,\n",
    "                              'requester_type': val_input_requester_type,\n",
    "                              'gender': val_input_gender}, \n",
    "                             {'service_title': val_service_title}, val_weights))\n",
    "        .batch(BATCH_SIZE)\n",
    "        .cache()\n",
    "        .prefetch(AUTO)\n",
    "    )\n",
    "    \n",
    "    return trn_input_ids.shape[0]//BATCH_SIZE, train_dataset, valid_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scheduler(epoch):\n",
    "    return LEARNING_RATE * 0.2**epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(pd.get_dummies(y))[[0, 1],].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    service_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_1')\n",
    "    service_title_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_1')\n",
    "    mfc_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_1')\n",
    "    internal_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_1')\n",
    "    external_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_1')\n",
    "    order_type_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_1')\n",
    "    department_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_1')\n",
    "    custom_service_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_1')\n",
    "    service_level_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_1')\n",
    "    is_subdep_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_1')\n",
    "    is_csid_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_1')\n",
    "    dayofweek_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_1')\n",
    "    day_part_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_1')\n",
    "    month_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_1')\n",
    "    week_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_1')\n",
    "    year_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_1')\n",
    "    person_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_1')\n",
    "    sole_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_1')\n",
    "    legal_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_1')\n",
    "    auto_ping_queue_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_1')\n",
    "    requester_type = tf.keras.layers.Input((1,), dtype=tf.int32, name='requester_type')\n",
    "    gender = tf.keras.layers.Input((1,), dtype=tf.int32, name='gender')\n",
    "    \n",
    "    service_1_embedding = tf.keras.layers.Embedding(len(X['service_1'].unique()), 11, input_length=1, name='service_1_embedding')(service_1)\n",
    "    service_title_1_embedding = tf.keras.layers.Embedding(len(X['service_title_1'].unique()), 50, input_length=1, name='service_title_1_embedding')(service_title_1)\n",
    "    mfc_1_embedding = tf.keras.layers.Embedding(len(X['mfc_1'].unique()), 50, input_length=1, name='mfc_1_embedding')(mfc_1)\n",
    "    internal_status_1_embedding = tf.keras.layers.Embedding(len(X['internal_status_1'].unique()), 5, input_length=1, name='internal_status_1_embedding')(internal_status_1)\n",
    "    external_status_1_embedding = tf.keras.layers.Embedding(len(X['external_status_1'].unique()), 14, input_length=1, name='external_status_1_embedding')(external_status_1)\n",
    "    order_type_1_embedding = tf.keras.layers.Embedding(len(X['order_type_1'].unique()), 2, input_length=1, name='order_type_1_embedding')(order_type_1)\n",
    "    department_id_1_embedding = tf.keras.layers.Embedding(len(X['department_id_1'].unique()), 46, input_length=1, name='department_id_1_embedding')(department_id_1)\n",
    "    custom_service_id_1_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_1'].unique()), 26, input_length=1, name='custom_service_id_1_embedding')(custom_service_id_1)\n",
    "    service_level_1_embedding = tf.keras.layers.Embedding(len(X['service_level_1'].unique()), 3, input_length=1, name='service_level_1_embedding')(service_level_1)\n",
    "    is_subdep_1_embedding = tf.keras.layers.Embedding(len(X['is_subdep_1'].unique()), 1, input_length=1, name='is_subdep_1_embedding')(is_subdep_1)\n",
    "    is_csid_1_embedding = tf.keras.layers.Embedding(len(X['is_csid_1'].unique()), 1, input_length=1, name='is_csid_1_embedding')(is_csid_1)\n",
    "    dayofweek_1_embedding = tf.keras.layers.Embedding(len(X['dayofweek_1'].unique()), 4, input_length=1, name='dayofweek_1_embedding')(dayofweek_1)\n",
    "    day_part_1_embedding = tf.keras.layers.Embedding(len(X['day_part_1'].unique()), 2, input_length=1, name='day_part_1_embedding')(day_part_1)\n",
    "    month_1_embedding = tf.keras.layers.Embedding(len(X['month_1'].unique()), 6, input_length=1, name='month_1_embedding')(month_1)\n",
    "    week_1_embedding = tf.keras.layers.Embedding(len(X['week_1'].unique()), 26, input_length=1, name='week_1_embedding')(week_1)\n",
    "    year_1_embedding = tf.keras.layers.Embedding(len(X['year_1'].unique()), 1, input_length=1, name='year_1_embedding')(year_1)\n",
    "    person_1_embedding = tf.keras.layers.Embedding(len(X['person_1'].unique()), 1, input_length=1, name='person_1_embedding')(person_1)\n",
    "    sole_1_embedding = tf.keras.layers.Embedding(len(X['sole_1'].unique()), 1, input_length=1, name='sole_1_embedding')(sole_1)\n",
    "    legal_1_embedding = tf.keras.layers.Embedding(len(X['legal_1'].unique()), 1, input_length=1, name='legal_1_embedding')(legal_1)\n",
    "    auto_ping_queue_1_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_1'].unique()), 1, input_length=1, name='auto_ping_queue_1_embedding')(auto_ping_queue_1)\n",
    "    requester_type_embedding = tf.keras.layers.Embedding(len(X['requester_type'].unique()), 2, input_length=1, name='requester_type_embedding')(requester_type)\n",
    "    gender_embedding = tf.keras.layers.Embedding(len(X['gender'].unique()), 1, input_length=1, name='gender_embedding')(gender)\n",
    "\n",
    "    concatenated = tf.keras.layers.Concatenate()([service_1_embedding, \n",
    "                                                  service_title_1_embedding,\n",
    "                                                 mfc_1_embedding,\n",
    "                                                 internal_status_1_embedding,\n",
    "                                                 external_status_1_embedding,\n",
    "                                                 order_type_1_embedding,\n",
    "                                                 department_id_1_embedding,\n",
    "                                                 custom_service_id_1_embedding,\n",
    "                                                 service_level_1_embedding,\n",
    "                                                 is_subdep_1_embedding,\n",
    "                                                 is_csid_1_embedding,\n",
    "                                                 dayofweek_1_embedding,\n",
    "                                                 day_part_1_embedding,\n",
    "                                                 month_1_embedding,\n",
    "                                                 year_1_embedding,\n",
    "                                                 person_1_embedding,\n",
    "                                                 sole_1_embedding,\n",
    "                                                 legal_1_embedding,\n",
    "                                                 auto_ping_queue_1_embedding,\n",
    "                                                 requester_type_embedding,\n",
    "                                                 gender_embedding])\n",
    "    #out = tf.keras.layers.Flatten()(concatenated)\n",
    "    \n",
    "    #out = tf.keras.layers.Dense(512, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Dense(256, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Dense(256, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Conv1D(128, 2, padding='same')(concatenated)\n",
    "    #out = tf.keras.layers.LeakyReLU()(out)\n",
    "    #out = tf.keras.layers.Conv1D(64, 2, padding='same')(out)\n",
    "    #out = tf.keras.layers.Flatten()(out)\n",
    "    conv_0 = tf.keras.layers.Conv1D(128, 3, padding='same', activation='relu')(concatenated)\n",
    "    conv_1 = tf.keras.layers.Conv1D(128, 2, padding='same', activation='relu')(concatenated)\n",
    "    conv_2 = tf.keras.layers.Conv1D(128, 6, padding='same', activation='relu')(concatenated)\n",
    "    conv_0 = tf.keras.layers.Conv1D(64, 3, padding='same', activation='relu')(conv_0)\n",
    "    conv_1 = tf.keras.layers.Conv1D(64, 2, padding='same', activation='relu')(conv_1)\n",
    "    conv_2 = tf.keras.layers.Conv1D(64, 6, padding='same', activation='relu')(conv_2)\n",
    "    concatenated_tensor = tf.keras.layers.Concatenate(axis=1)([conv_0, conv_1, conv_2])\n",
    "    out = tf.keras.layers.Flatten()(concatenated_tensor)\n",
    "    \n",
    "    out = tf.keras.layers.Dense(len(y.unique()), activation='softmax', name='service_title')(out)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=[service_1, \n",
    "                                          service_title_1, \n",
    "                                          mfc_1,\n",
    "                                         internal_status_1,\n",
    "                                         external_status_1,\n",
    "                                         order_type_1,\n",
    "                                         department_id_1,\n",
    "                                         custom_service_id_1,\n",
    "                                         service_level_1,\n",
    "                                         is_subdep_1,\n",
    "                                         is_csid_1,\n",
    "                                         dayofweek_1,\n",
    "                                         day_part_1,\n",
    "                                         month_1,\n",
    "                                         week_1,\n",
    "                                         year_1,\n",
    "                                         person_1,\n",
    "                                         sole_1,\n",
    "                                         legal_1,\n",
    "                                         auto_ping_queue_1,\n",
    "                                         requester_type,\n",
    "                                         gender], \n",
    "                                  outputs=out)\n",
    "\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_splits = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### FOLD 1\n",
      "#########################\n",
      "Epoch 1/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4328 - loss: 2.2617\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.44124, saving model to v5-learnedemb-0.h5\n",
      "2296/2296 [==============================] - 104s 45ms/step - accuracy: 0.4328 - loss: 2.2617 - val_accuracy: 0.4412 - val_loss: 2.2084 - lr: 0.0080\n",
      "Epoch 2/5\n",
      "2295/2296 [============================>.] - ETA: 0s - accuracy: 0.4498 - loss: 2.1385\n",
      "Epoch 00002: val_accuracy improved from 0.44124 to 0.45004, saving model to v5-learnedemb-0.h5\n",
      "2296/2296 [==============================] - 98s 43ms/step - accuracy: 0.4499 - loss: 2.1384 - val_accuracy: 0.4500 - val_loss: 2.1452 - lr: 0.0016\n",
      "Epoch 3/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4545 - loss: 2.1039\n",
      "Epoch 00003: val_accuracy improved from 0.45004 to 0.45179, saving model to v5-learnedemb-0.h5\n",
      "2296/2296 [==============================] - 97s 42ms/step - accuracy: 0.4545 - loss: 2.1039 - val_accuracy: 0.4518 - val_loss: 2.1328 - lr: 3.2000e-04\n",
      "Epoch 4/5\n",
      "2295/2296 [============================>.] - ETA: 0s - accuracy: 0.4559 - loss: 2.0938\n",
      "Epoch 00004: val_accuracy improved from 0.45179 to 0.45229, saving model to v5-learnedemb-0.h5\n",
      "2296/2296 [==============================] - 96s 42ms/step - accuracy: 0.4559 - loss: 2.0939 - val_accuracy: 0.4523 - val_loss: 2.1308 - lr: 6.4000e-05\n",
      "Epoch 5/5\n",
      "2295/2296 [============================>.] - ETA: 0s - accuracy: 0.4541 - loss: 2.1049\n",
      "Epoch 00003: val_accuracy improved from 0.45045 to 0.45250, saving model to v5-learnedemb-1.h5\n",
      "2296/2296 [==============================] - 96s 42ms/step - accuracy: 0.4541 - loss: 2.1049 - val_accuracy: 0.4525 - val_loss: 2.1294 - lr: 3.2000e-04\n",
      "Epoch 4/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4557 - loss: 2.0950\n",
      "Epoch 00004: val_accuracy improved from 0.45250 to 0.45278, saving model to v5-learnedemb-1.h5\n",
      "2296/2296 [==============================] - 96s 42ms/step - accuracy: 0.4557 - loss: 2.0950 - val_accuracy: 0.4528 - val_loss: 2.1275 - lr: 6.4000e-05\n",
      "Epoch 5/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4560 - loss: 2.0926\n",
      "Epoch 00005: val_accuracy improved from 0.45278 to 0.45285, saving model to v5-learnedemb-1.h5\n",
      "2296/2296 [==============================] - 96s 42ms/step - accuracy: 0.4560 - loss: 2.0926 - val_accuracy: 0.4528 - val_loss: 2.1272 - lr: 1.2800e-05\n",
      "#########################\n",
      "### FOLD 3\n",
      "#########################\n",
      "Epoch 1/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4329 - loss: 2.2601\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.43886, saving model to v5-learnedemb-2.h5\n",
      "2296/2296 [==============================] - 104s 45ms/step - accuracy: 0.4329 - loss: 2.2601 - val_accuracy: 0.4389 - val_loss: 2.2122 - lr: 0.0080\n",
      "Epoch 2/5\n",
      "2295/2296 [============================>.] - ETA: 0s - accuracy: 0.4496 - loss: 2.1367\n",
      "Epoch 00002: val_accuracy improved from 0.43886 to 0.45060, saving model to v5-learnedemb-2.h5\n",
      "2296/2296 [==============================] - 97s 42ms/step - accuracy: 0.4496 - loss: 2.1368 - val_accuracy: 0.4506 - val_loss: 2.1454 - lr: 0.0016\n",
      "Epoch 3/5\n",
      "2295/2296 [============================>.] - ETA: 0s - accuracy: 0.4545 - loss: 2.1021\n",
      "Epoch 00003: val_accuracy improved from 0.45060 to 0.45195, saving model to v5-learnedemb-2.h5\n",
      "2296/2296 [==============================] - 98s 43ms/step - accuracy: 0.4545 - loss: 2.1021 - val_accuracy: 0.4520 - val_loss: 2.1320 - lr: 3.2000e-04\n",
      "Epoch 4/5\n",
      " 121/2296 [>.............................] - ETA: 1:13 - accuracy: 0.4529 - loss: 2.1191\n",
      "Epoch 00002: val_accuracy improved from 0.43793 to 0.45006, saving model to v5-learnedemb-4.h5\n",
      "2296/2296 [==============================] - 98s 43ms/step - accuracy: 0.4495 - loss: 2.1377 - val_accuracy: 0.4501 - val_loss: 2.1419 - lr: 0.0016\n",
      "Epoch 3/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4539 - loss: 2.1028\n",
      "Epoch 00003: val_accuracy improved from 0.45006 to 0.45180, saving model to v5-learnedemb-4.h5\n",
      "2296/2296 [==============================] - 96s 42ms/step - accuracy: 0.4539 - loss: 2.1028 - val_accuracy: 0.4518 - val_loss: 2.1291 - lr: 3.2000e-04\n",
      "Epoch 4/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4554 - loss: 2.0927\n",
      "Epoch 00004: val_accuracy improved from 0.45180 to 0.45200, saving model to v5-learnedemb-4.h5\n",
      "2296/2296 [==============================] - 97s 42ms/step - accuracy: 0.4554 - loss: 2.0927 - val_accuracy: 0.4520 - val_loss: 2.1271 - lr: 6.4000e-05\n",
      "Epoch 5/5\n",
      "2296/2296 [==============================] - ETA: 0s - accuracy: 0.4558 - loss: 2.0902\n",
      "Epoch 00005: val_accuracy did not improve from 0.45200\n",
      "2296/2296 [==============================] - 97s 42ms/step - accuracy: 0.4558 - loss: 2.0902 - val_accuracy: 0.4519 - val_loss: 2.1267 - lr: 1.2800e-05\n"
     ]
    }
   ],
   "source": [
    "VER='v5'\n",
    "DISPLAY=1 # USE display=1 FOR INTERACTIVE\n",
    "\n",
    "service_title = np.zeros((X.shape[0], 1))\n",
    "\n",
    "skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777)\n",
    "\n",
    "\n",
    "for fold, (idxT, idxV) in enumerate(skf.split(X,y)):\n",
    "\n",
    "    print('#'*25)\n",
    "    print('### FOLD %i'%(fold+1))\n",
    "    print('#'*25)\n",
    "\n",
    "    # Cleaning everything\n",
    "    K.clear_session()\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "\n",
    "    # Building model\n",
    "    with strategy.scope():\n",
    "        model = build_model()\n",
    "\n",
    "    n_steps, trn_dataset, val_dataset = generate_dataset(idxT, idxV)\n",
    "\n",
    "    reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "\n",
    "    sv = tf.keras.callbacks.ModelCheckpoint(\n",
    "        '%s-learnedemb-%i.h5'%(VER,fold), monitor='val_accuracy', verbose=1, save_best_only=True,\n",
    "        save_weights_only=True, mode='auto', save_freq='epoch')\n",
    "\n",
    "    hist = model.fit(trn_dataset,\n",
    "                    epochs=EPOCHS, \n",
    "                    verbose=DISPLAY, \n",
    "                    callbacks=[sv, reduce_lr],\n",
    "                    validation_data=val_dataset)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}