Prediction (2 TF models).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making prediction based on 1 lag and 2 lags TF models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as sparse\n",
    "from scipy.sparse.linalg import spsolve\n",
    "import random\n",
    "import os\n",
    "import scipy.stats as ss\n",
    "import scipy\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from catboost import CatBoostClassifier, Pool, sum_models, to_classifier\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = pd.read_csv(\"../input/feature-creating-v2/train_3lags_v3.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df.loc[df['order_count'] == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#threshold = 0.0004\n",
    "#counts = df['service_title'].value_counts(normalize=True)\n",
    "#df2 = df.loc[df['service_title'].isin(counts[counts > threshold].index), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#d_134 = df2.loc[df2['service_title'] == 134][:121500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#d_98 = df2.loc[df2['service_title'] == 98][:121500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = df2.loc[df2['service_title'] != 134]\n",
    "#df2 = df2.loc[df2['service_title'] != 98]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = pd.concat([df2,d_134], axis=0)\n",
    "#df2 = pd.concat([df2,d_98], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2.to_csv('train_3lags_semibalanced.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(\"../input/irkutsk/train_3lags_semibalanced.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df2[['service_title', 'order_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.drop(['service',\n",
    "'service_title',\n",
    "'mfc',\n",
    "'internal_status',\n",
    "'external_status',\n",
    "'order_type',\n",
    "'department_id',\n",
    "'custom_service_id',\n",
    "'service_level',\n",
    "'is_subdep',\n",
    "'is_csid',\n",
    "'proc_time',\n",
    "'dayofweek',\n",
    "'day_part',\n",
    "'person',\n",
    "'sole',\n",
    "'legal',\n",
    "'auto_ping_queue',\n",
    "'win_count',\n",
    "'month',\n",
    "'week',\n",
    "'year'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df2.loc[df2['order_count'] > 1]\n",
    "df_test = df_test.loc[df_test['order_count'] > 1]\n",
    "df_train.reset_index(inplace=True)\n",
    "df_test.reset_index(inplace=True)\n",
    "df_train.drop(['index'], axis=1, inplace=True)\n",
    "df_test.drop(['index'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2','service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'order_type_3', 'department_id_3', 'custom_service_id_3',\n",
    "       'service_level_3', 'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', 'day_part_3', 'person_3', 'sole_3', 'legal_3',\n",
    "       'auto_ping_queue_3',\n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'department_id_2', 'month_2', 'week_2', 'year_2',\n",
    "       'custom_service_id_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'department_id_3', 'custom_service_id_3',\n",
    "       'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train.drop(['requester'], axis=1)\n",
    "y = df_test['service_title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.drop(['service_2',\n",
    "'service_title_2',\n",
    "'mfc_2',\n",
    "'internal_status_2',\n",
    "'external_status_2',\n",
    "'order_type_2',\n",
    "'department_id_2',\n",
    "'custom_service_id_2',\n",
    "'service_level_2',\n",
    "'is_subdep_2',\n",
    "'is_csid_2',\n",
    "'proc_time_2',\n",
    "'dayofweek_2',\n",
    "'day_part_2',\n",
    "'person_2',\n",
    "'sole_2',\n",
    "'legal_2',\n",
    "'auto_ping_queue_2',\n",
    "'win_count_2',\n",
    "'month_2',\n",
    "'week_2',\n",
    "'year_2'], axis=1 , inplace=True)\n",
    "\n",
    "X.drop(['service_3',\n",
    "'service_title_3',\n",
    "'mfc_3',\n",
    "'internal_status_3',\n",
    "'external_status_3',\n",
    "'order_type_3',\n",
    "'department_id_3',\n",
    "'custom_service_id_3',\n",
    "'service_level_3',\n",
    "'is_subdep_3',\n",
    "'is_csid_3',\n",
    "'proc_time_3',\n",
    "'dayofweek_3',\n",
    "'day_part_3',\n",
    "'person_3',\n",
    "'sole_3',\n",
    "'legal_3',\n",
    "'auto_ping_queue_3',\n",
    "'win_count_3',\n",
    "'month_3',\n",
    "'week_3',\n",
    "'year_3'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'requester_type', 'gender']\n",
    "\n",
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[cat] = X[cat].astype('Int64')\n",
    "X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[categorical] = X[categorical].fillna('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df):\n",
    "    \"\"\" \n",
    "    iterate through all the columns of a dataframe and \n",
    "    modify the data type to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage of dataframe is {:.2f}' \n",
    "                     'MB').format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max <\\\n",
    "                  np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max <\\\n",
    "                   np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max <\\\n",
    "                   np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max <\\\n",
    "                   np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max <\\\n",
    "                   np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max <\\\n",
    "                   np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage after optimization is: {:.2f}' \n",
    "                              'MB').format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) \n",
    "                                             / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = reduce_mem_usage(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X[categorical]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['person_1'] = X['person_1'].astype('int32')\n",
    "X['sole_1'] = X['sole_1'].astype('int32')\n",
    "X['legal_1'] = X['legal_1'].astype('int32')\n",
    "X['auto_ping_queue_1'] = X['auto_ping_queue_1'].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "labeling = []\n",
    "for col in X.columns:\n",
    "    d = pd.DataFrame()\n",
    "    d[col] = X[col].unique()\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.fit(d[col])\n",
    "    d[col+'_l'] = le.transform(d[col])\n",
    "    d = d.sort_values(by=[col+'_l']).reset_index(drop=True)\n",
    "    labeling.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame()\n",
    "d['service_title'] = y.unique()\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(d['service_title'])\n",
    "d['service_title'+'_l'] = le.transform(d['service_title'])\n",
    "d = d.sort_values(by=['service_title'+'_l']).reset_index(drop=True)\n",
    "labeling_y = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeling[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in X.columns:\n",
    "    X[col] = X[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.map(labeling_y.set_index('service_title').to_dict()['service_title'+'_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_l = labeling_y[:]\n",
    "weights_l['weights'] = 1\n",
    "\n",
    "weights = pd.DataFrame(y)\n",
    "weights = pd.merge(weights, weights_l, how='left', left_on='service_title', right_on='service_title_l')\n",
    "weights['weights'].loc[weights['service_title_y'] == 4] = 0.4\n",
    "weights['weights'].loc[weights['service_title_y'] == 603] = 0.6\n",
    "weights['weights'].loc[weights['service_title_y'] == 98] = 0.9\n",
    "weights['weights'].loc[weights['service_title_y'] == 134] = 0.87\n",
    "weights = np.array(weights['weights'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Detect hardware, return appropriate distribution strategy\n",
    "try:\n",
    "    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n",
    "    # set: this is always the case on Kaggle.\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    tpu = None\n",
    "\n",
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "BATCH_SIZE = 64 * strategy.num_replicas_in_sync\n",
    "LEARNING_RATE = 1e-3 * strategy.num_replicas_in_sync \n",
    "EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    service_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_1')\n",
    "    service_title_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_1')\n",
    "    mfc_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_1')\n",
    "    internal_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_1')\n",
    "    external_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_1')\n",
    "    order_type_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_1')\n",
    "    department_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_1')\n",
    "    custom_service_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_1')\n",
    "    service_level_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_1')\n",
    "    is_subdep_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_1')\n",
    "    is_csid_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_1')\n",
    "    dayofweek_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_1')\n",
    "    day_part_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_1')\n",
    "    month_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_1')\n",
    "    week_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_1')\n",
    "    year_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_1')\n",
    "    person_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_1')\n",
    "    sole_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_1')\n",
    "    legal_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_1')\n",
    "    auto_ping_queue_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_1')\n",
    "    requester_type = tf.keras.layers.Input((1,), dtype=tf.int32, name='requester_type')\n",
    "    gender = tf.keras.layers.Input((1,), dtype=tf.int32, name='gender')\n",
    "    \n",
    "    service_1_embedding = tf.keras.layers.Embedding(len(X['service_1'].unique()), 11, input_length=1, name='service_1_embedding')(service_1)\n",
    "    service_title_1_embedding = tf.keras.layers.Embedding(len(X['service_title_1'].unique()), 50, input_length=1, name='service_title_1_embedding')(service_title_1)\n",
    "    mfc_1_embedding = tf.keras.layers.Embedding(len(X['mfc_1'].unique()), 50, input_length=1, name='mfc_1_embedding')(mfc_1)\n",
    "    internal_status_1_embedding = tf.keras.layers.Embedding(len(X['internal_status_1'].unique()), 5, input_length=1, name='internal_status_1_embedding')(internal_status_1)\n",
    "    external_status_1_embedding = tf.keras.layers.Embedding(len(X['external_status_1'].unique()), 14, input_length=1, name='external_status_1_embedding')(external_status_1)\n",
    "    order_type_1_embedding = tf.keras.layers.Embedding(len(X['order_type_1'].unique()), 2, input_length=1, name='order_type_1_embedding')(order_type_1)\n",
    "    department_id_1_embedding = tf.keras.layers.Embedding(len(X['department_id_1'].unique()), 46, input_length=1, name='department_id_1_embedding')(department_id_1)\n",
    "    custom_service_id_1_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_1'].unique()), 26, input_length=1, name='custom_service_id_1_embedding')(custom_service_id_1)\n",
    "    service_level_1_embedding = tf.keras.layers.Embedding(len(X['service_level_1'].unique()), 3, input_length=1, name='service_level_1_embedding')(service_level_1)\n",
    "    is_subdep_1_embedding = tf.keras.layers.Embedding(len(X['is_subdep_1'].unique()), 1, input_length=1, name='is_subdep_1_embedding')(is_subdep_1)\n",
    "    is_csid_1_embedding = tf.keras.layers.Embedding(len(X['is_csid_1'].unique()), 1, input_length=1, name='is_csid_1_embedding')(is_csid_1)\n",
    "    dayofweek_1_embedding = tf.keras.layers.Embedding(len(X['dayofweek_1'].unique()), 4, input_length=1, name='dayofweek_1_embedding')(dayofweek_1)\n",
    "    day_part_1_embedding = tf.keras.layers.Embedding(len(X['day_part_1'].unique()), 2, input_length=1, name='day_part_1_embedding')(day_part_1)\n",
    "    month_1_embedding = tf.keras.layers.Embedding(len(X['month_1'].unique()), 6, input_length=1, name='month_1_embedding')(month_1)\n",
    "    week_1_embedding = tf.keras.layers.Embedding(len(X['week_1'].unique()), 26, input_length=1, name='week_1_embedding')(week_1)\n",
    "    year_1_embedding = tf.keras.layers.Embedding(len(X['year_1'].unique()), 1, input_length=1, name='year_1_embedding')(year_1)\n",
    "    person_1_embedding = tf.keras.layers.Embedding(len(X['person_1'].unique()), 1, input_length=1, name='person_1_embedding')(person_1)\n",
    "    sole_1_embedding = tf.keras.layers.Embedding(len(X['sole_1'].unique()), 1, input_length=1, name='sole_1_embedding')(sole_1)\n",
    "    legal_1_embedding = tf.keras.layers.Embedding(len(X['legal_1'].unique()), 1, input_length=1, name='legal_1_embedding')(legal_1)\n",
    "    auto_ping_queue_1_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_1'].unique()), 1, input_length=1, name='auto_ping_queue_1_embedding')(auto_ping_queue_1)\n",
    "    requester_type_embedding = tf.keras.layers.Embedding(len(X['requester_type'].unique()), 2, input_length=1, name='requester_type_embedding')(requester_type)\n",
    "    gender_embedding = tf.keras.layers.Embedding(len(X['gender'].unique()), 1, input_length=1, name='gender_embedding')(gender)\n",
    "\n",
    "    concatenated = tf.keras.layers.Concatenate()([service_1_embedding, \n",
    "                                                  service_title_1_embedding,\n",
    "                                                 mfc_1_embedding,\n",
    "                                                 internal_status_1_embedding,\n",
    "                                                 external_status_1_embedding,\n",
    "                                                 order_type_1_embedding,\n",
    "                                                 department_id_1_embedding,\n",
    "                                                 custom_service_id_1_embedding,\n",
    "                                                 service_level_1_embedding,\n",
    "                                                 is_subdep_1_embedding,\n",
    "                                                 is_csid_1_embedding,\n",
    "                                                 dayofweek_1_embedding,\n",
    "                                                 day_part_1_embedding,\n",
    "                                                 month_1_embedding,\n",
    "                                                 year_1_embedding,\n",
    "                                                 person_1_embedding,\n",
    "                                                 sole_1_embedding,\n",
    "                                                 legal_1_embedding,\n",
    "                                                 auto_ping_queue_1_embedding,\n",
    "                                                 requester_type_embedding,\n",
    "                                                 gender_embedding])\n",
    "\n",
    "    conv_0 = tf.keras.layers.Conv1D(128, 3, padding='same', activation='relu')(concatenated)\n",
    "    conv_1 = tf.keras.layers.Conv1D(128, 2, padding='same', activation='relu')(concatenated)\n",
    "    conv_2 = tf.keras.layers.Conv1D(128, 6, padding='same', activation='relu')(concatenated)\n",
    "    conv_0 = tf.keras.layers.Conv1D(64, 3, padding='same', activation='relu')(conv_0)\n",
    "    conv_1 = tf.keras.layers.Conv1D(64, 2, padding='same', activation='relu')(conv_1)\n",
    "    conv_2 = tf.keras.layers.Conv1D(64, 6, padding='same', activation='relu')(conv_2)\n",
    "    concatenated_tensor = tf.keras.layers.Concatenate(axis=1)([conv_0, conv_1, conv_2])\n",
    "    out = tf.keras.layers.Flatten()(concatenated_tensor)\n",
    "    \n",
    "    out = tf.keras.layers.Dense(len(y.unique()), activation='softmax', name='service_title')(out)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=[service_1, \n",
    "                                          service_title_1, \n",
    "                                          mfc_1,\n",
    "                                         internal_status_1,\n",
    "                                         external_status_1,\n",
    "                                         order_type_1,\n",
    "                                         department_id_1,\n",
    "                                         custom_service_id_1,\n",
    "                                         service_level_1,\n",
    "                                         is_subdep_1,\n",
    "                                         is_csid_1,\n",
    "                                         dayofweek_1,\n",
    "                                         day_part_1,\n",
    "                                         month_1,\n",
    "                                         week_1,\n",
    "                                         year_1,\n",
    "                                         person_1,\n",
    "                                         sole_1,\n",
    "                                         legal_1,\n",
    "                                         auto_ping_queue_1,\n",
    "                                         requester_type,\n",
    "                                         gender], \n",
    "                                  outputs=out)\n",
    "\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = [np.array(X_test['service_1']),\n",
    "np.array(X_test['service_title_1']),\n",
    "np.array(X_test['mfc_1']),\n",
    "np.array(X_test['internal_status_1']),\n",
    "np.array(X_test['external_status_1']),\n",
    "np.array(X_test['order_type_1']),\n",
    "np.array(X_test['department_id_1']),\n",
    "np.array(X_test['custom_service_id_1']),\n",
    "np.array(X_test['service_level_1']),\n",
    "np.array(X_test['is_subdep_1']),\n",
    "np.array(X_test['is_csid_1']),\n",
    "np.array(X_test['dayofweek_1']),\n",
    "np.array(X_test['day_part_1']),\n",
    "np.array(X_test['month_1']),\n",
    "np.array(X_test['week_1']),\n",
    "np.array(X_test['year_1']),\n",
    "np.array(X_test['person_1']),\n",
    "np.array(X_test['sole_1']),\n",
    "np.array(X_test['legal_1']),\n",
    "np.array(X_test['auto_ping_queue_1']),\n",
    "np.array(X_test['requester_type']),\n",
    "np.array(X_test['gender'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold_preds = np.zeros((X_test.shape[0], len(y.unique())))\n",
    "n_splits = 5\n",
    "for i in range(5):\n",
    "    print('#'*25)\n",
    "    print('### MODEL %i'%(i+1))\n",
    "    print('#'*25)\n",
    "    \n",
    "    K.clear_session()\n",
    "    model = build_model()\n",
    "    model.load_weights('../input/v55fold/v5-learnedemb-%i.h5'%i)\n",
    "\n",
    "    print('Predicting Test...')\n",
    "    preds = model.predict(inputs,verbose=True)\n",
    "    kfold_preds += preds/n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = []\n",
    "for k in range(y_test.shape[0]):\n",
    "    a = np.argmax(kfold_preds[k,])\n",
    "    y_pred.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_score(y_pred, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../input/feature-creating/train_3lags_v1.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab = pd.read_csv(\"../input/irkutsk/sample_submission.csv\")\n",
    "\n",
    "sab_pred = sab[['requester']]\n",
    "\n",
    "sab_pred = pd.merge(sab_pred, df, how='left', on='requester')\n",
    "\n",
    "sab_pred.drop(['service_3',\n",
    "'service_title_3',\n",
    "'mfc_3',\n",
    "'internal_status_3',\n",
    "'external_status_3',\n",
    "'order_type_3',\n",
    "'department_id_3',\n",
    "'custom_service_id_3',\n",
    "'service_level_3',\n",
    "'is_subdep_3',\n",
    "'is_csid_3',\n",
    "'proc_time_3',\n",
    "'dayofweek_3',\n",
    "'day_part_3',\n",
    "'person_3',\n",
    "'sole_3',\n",
    "'legal_3',\n",
    "'auto_ping_queue_3',\n",
    "'win_count_3', 'year_3', 'month_3', 'week_3'], axis=1 , inplace=True)\n",
    "\n",
    "sab_pred.drop(['requester'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = sab_pred.loc[sab_pred['order_count'] == 1].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred = sab_pred.iloc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.drop(['service_2',\n",
    "'service_title_2',\n",
    "'mfc_2',\n",
    "'internal_status_2',\n",
    "'external_status_2',\n",
    "'order_type_2',\n",
    "'department_id_2',\n",
    "'custom_service_id_2',\n",
    "'service_level_2',\n",
    "'is_subdep_2',\n",
    "'is_csid_2',\n",
    "'proc_time_2',\n",
    "'dayofweek_2',\n",
    "'day_part_2',\n",
    "'person_2',\n",
    "'sole_2',\n",
    "'legal_2',\n",
    "'auto_ping_queue_2',\n",
    "'win_count_2',\n",
    "'month_2',\n",
    "'week_2',\n",
    "'year_2'], axis=1 , inplace=True)\n",
    "\n",
    "sab_pred.drop(['service_1',\n",
    "'service_title_1',\n",
    "'mfc_1',\n",
    "'internal_status_1',\n",
    "'external_status_1',\n",
    "'order_type_1',\n",
    "'department_id_1',\n",
    "'custom_service_id_1',\n",
    "'service_level_1',\n",
    "'is_subdep_1',\n",
    "'is_csid_1',\n",
    "'proc_time_1',\n",
    "'dayofweek_1',\n",
    "'day_part_1',\n",
    "'person_1',\n",
    "'sole_1',\n",
    "'legal_1',\n",
    "'auto_ping_queue_1',\n",
    "'win_count_1',\n",
    "'month_1',\n",
    "'week_1',\n",
    "'year_1'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.drop(['proc_time', 'win_count', 'age', 'order_count'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_cols=X.columns\n",
    "sab_pred.rename(columns=dict(zip(sab_pred.columns, new_cols)),inplace=True)\n",
    "sab_pred[cat] = sab_pred[cat].astype('Int64')\n",
    "sab_pred[cat] = sab_pred[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['person_1'] = sab_pred['person_1'].map({True: 1, False:0})\n",
    "sab_pred['sole_1'] = sab_pred['sole_1'].map({True: 1, False:0})\n",
    "sab_pred['legal_1'] = sab_pred['legal_1'].map({True: 1, False:0})\n",
    "sab_pred['auto_ping_queue_1'] = sab_pred['auto_ping_queue_1'].map({True: 1, False:0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in sab_pred.columns:\n",
    "    sab_pred[col] = sab_pred[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toint = ['service_1', 'service_title_1', 'mfc_1', 'order_type_1', 'person_1', 'sole_1', 'auto_ping_queue_1', 'day_part_1',\n",
    "       'internal_status_1', 'external_status_1', 'service_level_1', 'legal_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[toint] = sab_pred[toint].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['order_type_1'] = sab_pred['order_type_1'].astype('category')\n",
    "sab_pred['service_level_1'] = sab_pred['service_level_1'].astype('category')\n",
    "sab_pred['day_part_1'] = sab_pred['day_part_1'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs_t = [np.array(sab_pred['service_1']),\n",
    "np.array(sab_pred['service_title_1']),\n",
    "np.array(sab_pred['mfc_1']),\n",
    "np.array(sab_pred['internal_status_1']),\n",
    "np.array(sab_pred['external_status_1']),\n",
    "np.array(sab_pred['order_type_1']),\n",
    "np.array(sab_pred['department_id_1']),\n",
    "np.array(sab_pred['custom_service_id_1']),\n",
    "np.array(sab_pred['service_level_1']),\n",
    "np.array(sab_pred['is_subdep_1']),\n",
    "np.array(sab_pred['is_csid_1']),\n",
    "np.array(sab_pred['dayofweek_1']),\n",
    "np.array(sab_pred['day_part_1']),\n",
    "np.array(sab_pred['month_1']),\n",
    "np.array(sab_pred['week_1']),\n",
    "np.array(sab_pred['year_1']),\n",
    "np.array(sab_pred['person_1']),\n",
    "np.array(sab_pred['sole_1']),\n",
    "np.array(sab_pred['legal_1']),\n",
    "np.array(sab_pred['auto_ping_queue_1']),\n",
    "np.array(sab_pred['requester_type']),\n",
    "np.array(sab_pred['gender'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold_preds = np.zeros((sab_pred.shape[0], len(y.unique())))\n",
    "n_splits = 5\n",
    "for i in range(5):\n",
    "    print('#'*25)\n",
    "    print('### MODEL %i'%(i+1))\n",
    "    print('#'*25)\n",
    "    \n",
    "    K.clear_session()\n",
    "    model = build_model()\n",
    "    model.load_weights('../input/v55fold/v5-learnedemb-%i.h5'%i)\n",
    "\n",
    "    print('Predicting Test...')\n",
    "    preds = model.predict(inputs_t,verbose=True)\n",
    "    kfold_preds += preds/n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = []\n",
    "for k in range(sab_pred.shape[0]):\n",
    "    a = np.argmax(kfold_preds[k,])\n",
    "    y_pred.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab['service_title'].iloc[idx] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab['service_title'].iloc[idx] = sab['service_title'].iloc[idx].map(labeling_y.set_index('service_title_l').to_dict()['service_title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(\"../input/irkutsk/train_3lags_semibalanced.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.drop(['service_3', 'service_title_3', 'mfc_3', 'internal_status_3',\n",
    "       'external_status_3', 'order_type_3', 'department_id_3',\n",
    "       'custom_service_id_3', 'service_level_3', 'is_subdep_3', 'is_csid_3',\n",
    "       'month_3', 'week_3', 'year_3', 'dayofweek_3', 'day_part_3', 'person_3',\n",
    "       'sole_3', 'legal_3', 'auto_ping_queue_3'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.drop(['proc_time_3', 'win_count_3'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df2[['service_title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df2.drop(['service_title', 'order_count'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
    "       'external_status_1', 'order_type_1', 'department_id_1',\n",
    "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
    "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
    "       'sole_1', 'legal_1', 'auto_ping_queue_1', 'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2',\n",
    "       'year_2', 'legal_2', 'auto_ping_queue_2', 'requester_type', 'gender',\n",
    "       'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.reset_index(inplace=True)\n",
    "df_test.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.drop(['index'], axis=1, inplace=True)\n",
    "df_test.drop(['index'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2','service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'order_type_3', 'department_id_3', 'custom_service_id_3',\n",
    "       'service_level_3', 'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', 'day_part_3', 'person_3', 'sole_3', 'legal_3',\n",
    "       'auto_ping_queue_3',\n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'department_id_2', 'month_2', 'week_2', 'year_2',\n",
    "       'custom_service_id_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'department_id_3', 'custom_service_id_3',\n",
    "       'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train\n",
    "y = df_test['service_title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1','service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2',\n",
    "       'requester_type', 'gender']\n",
    "\n",
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'service_2', 'service_title_2', 'mfc_2',\n",
    "       'internal_status_2', 'external_status_2',\n",
    "       'department_id_2', 'custom_service_id_2', 'month_2', 'week_2', 'year_2',\n",
    "       'is_subdep_2', 'is_csid_2', 'dayofweek_2', 'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[cat] = X[cat].astype('Int64')\n",
    "X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[categorical] = X[categorical].fillna('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = reduce_mem_usage(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['person_1'] = X['person_1'].astype('int32')\n",
    "X['sole_1'] = X['sole_1'].astype('int32')\n",
    "X['legal_1'] = X['legal_1'].astype('int32')\n",
    "X['auto_ping_queue_1'] = X['auto_ping_queue_1'].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lag_2 = ['service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2']\n",
    "X[lag_2] = X[lag_2].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "labeling = []\n",
    "for col in X[categorical].columns:\n",
    "    d = pd.DataFrame()\n",
    "    d[col] = X[col].unique()\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.fit(d[col])\n",
    "    d[col+'_l'] = le.transform(d[col])\n",
    "    d = d.sort_values(by=[col+'_l']).reset_index(drop=True)\n",
    "    labeling.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X[['age']])\n",
    "X[['age']] = scaler.transform(X[['age']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame()\n",
    "d['service_title'] = y.unique()\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(d['service_title'])\n",
    "d['service_title'+'_l'] = le.transform(d['service_title'])\n",
    "d = d.sort_values(by=['service_title'+'_l']).reset_index(drop=True)\n",
    "labeling_y = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in X[categorical].columns:\n",
    "    X[col] = X[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.map(labeling_y.set_index('service_title').to_dict()['service_title'+'_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    service_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_1')\n",
    "    service_title_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_1')\n",
    "    mfc_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_1')\n",
    "    internal_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_1')\n",
    "    external_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_1')\n",
    "    order_type_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_1')\n",
    "    department_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_1')\n",
    "    custom_service_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_1')\n",
    "    service_level_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_1')\n",
    "    is_subdep_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_1')\n",
    "    is_csid_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_1')\n",
    "    dayofweek_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_1')\n",
    "    day_part_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_1')\n",
    "    month_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_1')\n",
    "    week_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_1')\n",
    "    year_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_1')\n",
    "    person_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_1')\n",
    "    sole_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_1')\n",
    "    legal_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_1')\n",
    "    auto_ping_queue_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_1')\n",
    "    \n",
    "    service_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_2')\n",
    "    service_title_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_2')\n",
    "    mfc_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_2')\n",
    "    internal_status_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_2')\n",
    "    external_status_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_2')\n",
    "    order_type_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_2')\n",
    "    department_id_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_2')\n",
    "    custom_service_id_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_2')\n",
    "    service_level_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_2')\n",
    "    is_subdep_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_2')\n",
    "    is_csid_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_2')\n",
    "    dayofweek_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_2')\n",
    "    day_part_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_2')\n",
    "    month_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_2')\n",
    "    week_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_2')\n",
    "    year_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_2')\n",
    "    person_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_2')\n",
    "    sole_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_2')\n",
    "    legal_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_2')\n",
    "    auto_ping_queue_2 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_2')\n",
    "    requester_type = tf.keras.layers.Input((1,), dtype=tf.int32, name='requester_type')\n",
    "    gender = tf.keras.layers.Input((1,), dtype=tf.int32, name='gender')\n",
    "    age = tf.keras.layers.Input((1,), dtype=tf.float64, name='age')\n",
    "    \n",
    "    service_1_embedding = tf.keras.layers.Embedding(len(X['service_1'].unique()), 11, input_length=1, name='service_1_embedding')(service_1)\n",
    "    service_title_1_embedding = tf.keras.layers.Embedding(len(X['service_title_1'].unique()), 50, input_length=1, name='service_title_1_embedding')(service_title_1)\n",
    "    mfc_1_embedding = tf.keras.layers.Embedding(len(X['mfc_1'].unique()), 50, input_length=1, name='mfc_1_embedding')(mfc_1)\n",
    "    internal_status_1_embedding = tf.keras.layers.Embedding(len(X['internal_status_1'].unique()), 5, input_length=1, name='internal_status_1_embedding')(internal_status_1)\n",
    "    external_status_1_embedding = tf.keras.layers.Embedding(len(X['external_status_1'].unique()), 14, input_length=1, name='external_status_1_embedding')(external_status_1)\n",
    "    order_type_1_embedding = tf.keras.layers.Embedding(len(X['order_type_1'].unique()), 2, input_length=1, name='order_type_1_embedding')(order_type_1)\n",
    "    department_id_1_embedding = tf.keras.layers.Embedding(len(X['department_id_1'].unique()), 46, input_length=1, name='department_id_1_embedding')(department_id_1)\n",
    "    custom_service_id_1_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_1'].unique()), 26, input_length=1, name='custom_service_id_1_embedding')(custom_service_id_1)\n",
    "    service_level_1_embedding = tf.keras.layers.Embedding(len(X['service_level_1'].unique()), 3, input_length=1, name='service_level_1_embedding')(service_level_1)\n",
    "    is_subdep_1_embedding = tf.keras.layers.Embedding(len(X['is_subdep_1'].unique()), 1, input_length=1, name='is_subdep_1_embedding')(is_subdep_1)\n",
    "    is_csid_1_embedding = tf.keras.layers.Embedding(len(X['is_csid_1'].unique()), 1, input_length=1, name='is_csid_1_embedding')(is_csid_1)\n",
    "    dayofweek_1_embedding = tf.keras.layers.Embedding(len(X['dayofweek_1'].unique()), 4, input_length=1, name='dayofweek_1_embedding')(dayofweek_1)\n",
    "    day_part_1_embedding = tf.keras.layers.Embedding(len(X['day_part_1'].unique()), 2, input_length=1, name='day_part_1_embedding')(day_part_1)\n",
    "    month_1_embedding = tf.keras.layers.Embedding(len(X['month_1'].unique()), 6, input_length=1, name='month_1_embedding')(month_1)\n",
    "    week_1_embedding = tf.keras.layers.Embedding(len(X['week_1'].unique()), 26, input_length=1, name='week_1_embedding')(week_1)\n",
    "    year_1_embedding = tf.keras.layers.Embedding(len(X['year_1'].unique()), 1, input_length=1, name='year_1_embedding')(year_1)\n",
    "    person_1_embedding = tf.keras.layers.Embedding(len(X['person_1'].unique()), 1, input_length=1, name='person_1_embedding')(person_1)\n",
    "    sole_1_embedding = tf.keras.layers.Embedding(len(X['sole_1'].unique()), 1, input_length=1, name='sole_1_embedding')(sole_1)\n",
    "    legal_1_embedding = tf.keras.layers.Embedding(len(X['legal_1'].unique()), 1, input_length=1, name='legal_1_embedding')(legal_1)\n",
    "    auto_ping_queue_1_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_1'].unique()), 1, input_length=1, name='auto_ping_queue_1_embedding')(auto_ping_queue_1)\n",
    "    \n",
    "    service_2_embedding = tf.keras.layers.Embedding(len(X['service_2'].unique()), 11, input_length=1, name='service_2_embedding')(service_2)\n",
    "    service_title_2_embedding = tf.keras.layers.Embedding(len(X['service_title_2'].unique()), 50, input_length=1, name='service_title_2_embedding')(service_title_2)\n",
    "    mfc_2_embedding = tf.keras.layers.Embedding(len(X['mfc_2'].unique()), 50, input_length=1, name='mfc_2_embedding')(mfc_2)\n",
    "    internal_status_2_embedding = tf.keras.layers.Embedding(len(X['internal_status_2'].unique()), 5, input_length=1, name='internal_status_2_embedding')(internal_status_2)\n",
    "    external_status_2_embedding = tf.keras.layers.Embedding(len(X['external_status_2'].unique()), 14, input_length=1, name='external_status_2_embedding')(external_status_2)\n",
    "    order_type_2_embedding = tf.keras.layers.Embedding(len(X['order_type_2'].unique()), 2, input_length=1, name='order_type_2_embedding')(order_type_2)\n",
    "    department_id_2_embedding = tf.keras.layers.Embedding(len(X['department_id_2'].unique()), 46, input_length=1, name='department_id_2_embedding')(department_id_2)\n",
    "    custom_service_id_2_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_2'].unique()), 26, input_length=1, name='custom_service_id_2_embedding')(custom_service_id_2)\n",
    "    service_level_2_embedding = tf.keras.layers.Embedding(len(X['service_level_2'].unique()), 3, input_length=1, name='service_level_2_embedding')(service_level_2)\n",
    "    is_subdep_2_embedding = tf.keras.layers.Embedding(len(X['is_subdep_2'].unique()), 1, input_length=1, name='is_subdep_2_embedding')(is_subdep_2)\n",
    "    is_csid_2_embedding = tf.keras.layers.Embedding(len(X['is_csid_2'].unique()), 1, input_length=1, name='is_csid_2_embedding')(is_csid_2)\n",
    "    dayofweek_2_embedding = tf.keras.layers.Embedding(len(X['dayofweek_2'].unique()), 4, input_length=1, name='dayofweek_2_embedding')(dayofweek_2)\n",
    "    day_part_2_embedding = tf.keras.layers.Embedding(len(X['day_part_2'].unique()), 2, input_length=1, name='day_part_2_embedding')(day_part_2)\n",
    "    month_2_embedding = tf.keras.layers.Embedding(len(X['month_2'].unique()), 6, input_length=1, name='month_2_embedding')(month_2)\n",
    "    week_2_embedding = tf.keras.layers.Embedding(len(X['week_2'].unique()), 26, input_length=1, name='week_2_embedding')(week_2)\n",
    "    year_2_embedding = tf.keras.layers.Embedding(len(X['year_2'].unique()), 1, input_length=1, name='year_2_embedding')(year_2)\n",
    "    person_2_embedding = tf.keras.layers.Embedding(len(X['person_2'].unique()), 1, input_length=1, name='person_2_embedding')(person_2)\n",
    "    sole_2_embedding = tf.keras.layers.Embedding(len(X['sole_2'].unique()), 1, input_length=1, name='sole_2_embedding')(sole_2)\n",
    "    legal_2_embedding = tf.keras.layers.Embedding(len(X['legal_2'].unique()), 1, input_length=1, name='legal_2_embedding')(legal_2)\n",
    "    auto_ping_queue_2_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_2'].unique()), 1, input_length=1, name='auto_ping_queue_2_embedding')(auto_ping_queue_2)\n",
    "    \n",
    "    requester_type_embedding = tf.keras.layers.Embedding(len(X['requester_type'].unique()), 2, input_length=1, name='requester_type_embedding')(requester_type)\n",
    "    gender_embedding = tf.keras.layers.Embedding(len(X['gender'].unique()), 1, input_length=1, name='gender_embedding')(gender)\n",
    "    age_reshape = tf.keras.layers.Reshape((1, 1), name='age_reshape')(age)\n",
    "\n",
    "    concatenated = tf.keras.layers.Concatenate()([service_1_embedding, \n",
    "                                                  service_title_1_embedding,\n",
    "                                                 mfc_1_embedding,\n",
    "                                                 internal_status_1_embedding,\n",
    "                                                 external_status_1_embedding,\n",
    "                                                 order_type_1_embedding,\n",
    "                                                 department_id_1_embedding,\n",
    "                                                 custom_service_id_1_embedding,\n",
    "                                                 service_level_1_embedding,\n",
    "                                                 is_subdep_1_embedding,\n",
    "                                                 is_csid_1_embedding,\n",
    "                                                 dayofweek_1_embedding,\n",
    "                                                 day_part_1_embedding,\n",
    "                                                 month_1_embedding,\n",
    "                                                 year_1_embedding,\n",
    "                                                 person_1_embedding,\n",
    "                                                 sole_1_embedding,\n",
    "                                                 legal_1_embedding,\n",
    "                                                 auto_ping_queue_1_embedding,\n",
    "                                                 service_2_embedding, \n",
    "                                                 service_title_2_embedding,\n",
    "                                                 mfc_2_embedding,\n",
    "                                                 internal_status_2_embedding,\n",
    "                                                 external_status_2_embedding,\n",
    "                                                 order_type_2_embedding,\n",
    "                                                 department_id_2_embedding,\n",
    "                                                 custom_service_id_2_embedding,\n",
    "                                                 service_level_2_embedding,\n",
    "                                                 is_subdep_2_embedding,\n",
    "                                                 is_csid_2_embedding,\n",
    "                                                 dayofweek_2_embedding,\n",
    "                                                 day_part_2_embedding,\n",
    "                                                 month_2_embedding,\n",
    "                                                 year_2_embedding,\n",
    "                                                 person_2_embedding,\n",
    "                                                 sole_2_embedding,\n",
    "                                                 legal_2_embedding,\n",
    "                                                 auto_ping_queue_2_embedding,\n",
    "                                                 requester_type_embedding,\n",
    "                                                 gender_embedding,\n",
    "                                                 age_reshape])\n",
    "\n",
    "    conv_0 = tf.keras.layers.Conv1D(256, 3, padding='same', activation='relu')(concatenated)\n",
    "    conv_1 = tf.keras.layers.Conv1D(256, 2, padding='same', activation='relu')(concatenated)\n",
    "    conv_2 = tf.keras.layers.Conv1D(256, 6, padding='same', activation='relu')(concatenated)\n",
    "    conv_0 = tf.keras.layers.Conv1D(128, 3, padding='same', activation='relu')(conv_0)\n",
    "    conv_1 = tf.keras.layers.Conv1D(128, 2, padding='same', activation='relu')(conv_1)\n",
    "    conv_2 = tf.keras.layers.Conv1D(128, 6, padding='same', activation='relu')(conv_2)\n",
    "    concatenated_tensor = tf.keras.layers.Concatenate(axis=1)([conv_0, conv_1, conv_2])\n",
    "    out = tf.keras.layers.Flatten()(concatenated_tensor)\n",
    "    \n",
    "    out = tf.keras.layers.Dropout(0.1)(out)\n",
    "    \n",
    "    out = tf.keras.layers.Dense(len(y.unique()), activation='softmax', name='service_title')(out)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=[service_1, \n",
    "                                          service_title_1, \n",
    "                                          mfc_1,\n",
    "                                         internal_status_1,\n",
    "                                         external_status_1,\n",
    "                                         order_type_1,\n",
    "                                         department_id_1,\n",
    "                                         custom_service_id_1,\n",
    "                                         service_level_1,\n",
    "                                         is_subdep_1,\n",
    "                                         is_csid_1,\n",
    "                                         dayofweek_1,\n",
    "                                         day_part_1,\n",
    "                                         month_1,\n",
    "                                         week_1,\n",
    "                                         year_1,\n",
    "                                         person_1,\n",
    "                                         sole_1,\n",
    "                                         legal_1,\n",
    "                                         auto_ping_queue_1,\n",
    "                                         service_2, \n",
    "                                         service_title_2, \n",
    "                                         mfc_2,\n",
    "                                         internal_status_2,\n",
    "                                         external_status_2,\n",
    "                                         order_type_2,\n",
    "                                         department_id_2,\n",
    "                                         custom_service_id_2,\n",
    "                                         service_level_2,\n",
    "                                         is_subdep_2,\n",
    "                                         is_csid_2,\n",
    "                                         dayofweek_2,\n",
    "                                         day_part_2,\n",
    "                                         month_2,\n",
    "                                         week_2,\n",
    "                                         year_2,\n",
    "                                         person_2,\n",
    "                                         sole_2,\n",
    "                                         legal_2,\n",
    "                                         auto_ping_queue_2,\n",
    "                                         requester_type,\n",
    "                                         gender,\n",
    "                                         age], \n",
    "                                  outputs=out)\n",
    "\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred = sab[['requester']]\n",
    "\n",
    "sab_pred = pd.merge(sab_pred, df, how='left', on='requester')\n",
    "\n",
    "sab_pred.drop(['service_3',\n",
    "'service_title_3',\n",
    "'mfc_3',\n",
    "'internal_status_3',\n",
    "'external_status_3',\n",
    "'order_type_3',\n",
    "'department_id_3',\n",
    "'custom_service_id_3',\n",
    "'service_level_3',\n",
    "'is_subdep_3',\n",
    "'is_csid_3',\n",
    "'proc_time_3',\n",
    "'dayofweek_3',\n",
    "'day_part_3',\n",
    "'person_3',\n",
    "'sole_3',\n",
    "'legal_3',\n",
    "'auto_ping_queue_3',\n",
    "'win_count_3', 'year_3', 'month_3', 'week_3'], axis=1 , inplace=True)\n",
    "\n",
    "sab_pred.drop(['requester'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = sab_pred.loc[sab_pred['order_count'] > 1].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred = sab_pred.iloc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.drop(['service_2',\n",
    "'service_title_2',\n",
    "'mfc_2',\n",
    "'internal_status_2',\n",
    "'external_status_2',\n",
    "'order_type_2',\n",
    "'department_id_2',\n",
    "'custom_service_id_2',\n",
    "'service_level_2',\n",
    "'is_subdep_2',\n",
    "'is_csid_2',\n",
    "'proc_time_2',\n",
    "'dayofweek_2',\n",
    "'day_part_2',\n",
    "'person_2',\n",
    "'sole_2',\n",
    "'legal_2',\n",
    "'auto_ping_queue_2',\n",
    "'win_count_2',\n",
    "'month_2',\n",
    "'week_2',\n",
    "'year_2'], axis=1 , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.drop(['proc_time', 'win_count', 'proc_time_1', 'win_count_1', 'order_count'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred = sab_pred[['service', 'service_title', 'mfc', 'internal_status',\n",
    "       'external_status', 'order_type', 'department_id',\n",
    "       'custom_service_id', 'service_level', 'is_subdep', 'is_csid',\n",
    "       'dayofweek', 'day_part', 'month', 'week', 'year', 'person',\n",
    "       'sole', 'legal', 'auto_ping_queue', 'service_1',\n",
    "       'service_title_1', 'mfc_1', 'internal_status_1', 'external_status_1',\n",
    "       'order_type_1', 'department_id_1', 'custom_service_id_1',\n",
    "       'service_level_1', 'is_subdep_1', 'is_csid_1', 'dayofweek_1',\n",
    "       'day_part_1', 'person_1', 'sole_1', 'month_1', 'week_1', 'year_1',\n",
    "       'legal_1', 'auto_ping_queue_1', 'requester_type', 'gender', 'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_cols=X.columns\n",
    "sab_pred.rename(columns=dict(zip(sab_pred.columns, new_cols)),inplace=True)\n",
    "sab_pred[cat] = sab_pred[cat].astype('Int64')\n",
    "sab_pred[cat] = sab_pred[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['person_1'] = sab_pred['person_1'].map({True: 1, False:0})\n",
    "sab_pred['sole_1'] = sab_pred['sole_1'].map({True: 1, False:0})\n",
    "sab_pred['legal_1'] = sab_pred['legal_1'].map({True: 1, False:0})\n",
    "sab_pred['auto_ping_queue_1'] = sab_pred['auto_ping_queue_1'].map({True: 1, False:0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['person_2'] = sab_pred['person_2'].map({True: '1.0', False: '0.0'})\n",
    "sab_pred['sole_2'] = sab_pred['sole_2'].map({True: '1.0', False:'0.0'})\n",
    "sab_pred['legal_2'] = sab_pred['legal_2'].map({True: '1.0', False:'0.0'})\n",
    "sab_pred['auto_ping_queue_2'] = sab_pred['auto_ping_queue_2'].map({True: '1.0', False:'0.0'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
    "       'external_status_1', 'order_type_1', 'department_id_1',\n",
    "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
    "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
    "       'sole_1', 'legal_1', 'auto_ping_queue_1']] = sab_pred[['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
    "       'external_status_1', 'order_type_1', 'department_id_1',\n",
    "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
    "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
    "       'sole_1', 'legal_1', 'auto_ping_queue_1']].fillna(sab_pred.mode().iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[['service_2',\n",
    "       'service_title_2', 'mfc_2', 'internal_status_2', 'external_status_2',\n",
    "       'order_type_2', 'department_id_2', 'custom_service_id_2',\n",
    "       'service_level_2', 'is_subdep_2', 'is_csid_2', 'dayofweek_2',\n",
    "       'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2']] = sab_pred[['service_2',\n",
    "       'service_title_2', 'mfc_2', 'internal_status_2', 'external_status_2',\n",
    "       'order_type_2', 'department_id_2', 'custom_service_id_2',\n",
    "       'service_level_2', 'is_subdep_2', 'is_csid_2', 'dayofweek_2',\n",
    "       'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2']].fillna('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['person_1'] = sab_pred['person_1'].astype('int64')\n",
    "sab_pred['sole_1'] = sab_pred['sole_1'].astype('int64')\n",
    "sab_pred['legal_1'] = sab_pred['legal_1'].astype('int64')\n",
    "sab_pred['auto_ping_queue_1'] = sab_pred['auto_ping_queue_1'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[lag_2] = sab_pred[lag_2].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in sab_pred[categorical].columns:\n",
    "    sab_pred[col] = sab_pred[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred.fillna(sab_pred.mode().iloc[0], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toint = ['service_1', 'service_title_1', 'mfc_1', 'order_type_1', 'person_1', 'sole_1', 'auto_ping_queue_1', 'day_part_1',\n",
    "       'internal_status_1', 'external_status_1', 'service_level_1', 'legal_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'service_title_2',\n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[toint] = sab_pred[toint].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['order_type_1'] = sab_pred['order_type_1'].astype('category')\n",
    "sab_pred['service_level_1'] = sab_pred['service_level_1'].astype('category')\n",
    "sab_pred['day_part_1'] = sab_pred['day_part_1'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[toint] = sab_pred[toint].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred['service_level_2'] = sab_pred['service_level_2'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[['order_type_1', 'service_level_1', 'day_part_1']] = sab_pred[['order_type_1', 'service_level_1', 'day_part_1']].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab_pred[['age']] = scaler.transform(sab_pred[['age']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs_t = [np.array(sab_pred['service_1']),\n",
    "np.array(sab_pred['service_title_1']),\n",
    "np.array(sab_pred['mfc_1']),\n",
    "np.array(sab_pred['internal_status_1']),\n",
    "np.array(sab_pred['external_status_1']),\n",
    "np.array(sab_pred['order_type_1']),\n",
    "np.array(sab_pred['department_id_1']),\n",
    "np.array(sab_pred['custom_service_id_1']),\n",
    "np.array(sab_pred['service_level_1']),\n",
    "np.array(sab_pred['is_subdep_1']),\n",
    "np.array(sab_pred['is_csid_1']),\n",
    "np.array(sab_pred['dayofweek_1']),\n",
    "np.array(sab_pred['day_part_1']),\n",
    "np.array(sab_pred['month_1']),\n",
    "np.array(sab_pred['week_1']),\n",
    "np.array(sab_pred['year_1']),\n",
    "np.array(sab_pred['person_1']),\n",
    "np.array(sab_pred['sole_1']),\n",
    "np.array(sab_pred['legal_1']),\n",
    "np.array(sab_pred['auto_ping_queue_1']),\n",
    "np.array(sab_pred['service_2']),\n",
    "np.array(sab_pred['service_title_2']),\n",
    "np.array(sab_pred['mfc_2']),\n",
    "np.array(sab_pred['internal_status_2']),\n",
    "np.array(sab_pred['external_status_2']),\n",
    "np.array(sab_pred['order_type_2']),\n",
    "np.array(sab_pred['department_id_2']),\n",
    "np.array(sab_pred['custom_service_id_2']),\n",
    "np.array(sab_pred['service_level_2']),\n",
    "np.array(sab_pred['is_subdep_2']),\n",
    "np.array(sab_pred['is_csid_2']),\n",
    "np.array(sab_pred['dayofweek_2']),\n",
    "np.array(sab_pred['day_part_2']),\n",
    "np.array(sab_pred['month_2']),\n",
    "np.array(sab_pred['week_2']),\n",
    "np.array(sab_pred['year_2']),\n",
    "np.array(sab_pred['person_2']),\n",
    "np.array(sab_pred['sole_2']),\n",
    "np.array(sab_pred['legal_2']),\n",
    "np.array(sab_pred['auto_ping_queue_2']),\n",
    "np.array(sab_pred['requester_type']),\n",
    "np.array(sab_pred['gender']),\n",
    "np.array(sab_pred['age'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold_preds = np.zeros((sab_pred.shape[0], labeling_y.shape[0]))\n",
    "n_splits = 5\n",
    "for i in range(5):\n",
    "    print('#'*25)\n",
    "    print('### MODEL %i'%(i+1))\n",
    "    print('#'*25)\n",
    "    \n",
    "    K.clear_session()\n",
    "    model = build_model()\n",
    "    model.load_weights('../input/categorical-embedding-v4-2lags/v11-learnedemb-%i.h5'%i)\n",
    "\n",
    "    print('Predicting Test...')\n",
    "    preds = model.predict(inputs_t,verbose=True)\n",
    "    kfold_preds += preds/n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = []\n",
    "for k in range(sab_pred.shape[0]):\n",
    "    a = np.argmax(kfold_preds[k,])\n",
    "    y_pred.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab['service_title'].iloc[idx] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab['service_title'].iloc[idx] = sab['service_title'].iloc[idx].map(labeling_y.set_index('service_title_l').to_dict()['service_title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab['service_title'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sab.to_csv('nn_5fold_2models_0.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}