get-embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting embeddings for categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as sparse\n",
    "from scipy.sparse.linalg import spsolve\n",
    "import random\n",
    "import os\n",
    "import scipy.stats as ss\n",
    "import scipy\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from catboost import CatBoostClassifier, Pool, sum_models, to_classifier\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = pd.read_csv(\"../input/irkutsk/train_3lags_semibalanced.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = pd.read_csv(\"../input/irkutsk/train_3lags_v4.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2.drop(['Unnamed: 0'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2 = df2.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_clustered = pd.read_csv(\"../input/irkutsk/df_train_clustered.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = X_clustered[['service_title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_test = df2[['service_title', 'order_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = X_clustered.drop(['index', 'age', 'cluster', 'service_title'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.reset_index(inplace=True)\n",
    "df_test.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:3997: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    }
   ],
   "source": [
    "df_train.drop(['index'], axis=1, inplace=True)\n",
    "df_test.drop(['index'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mask = (df_train_mask['order_count'] == 5)\n",
    "#z_valid = df_train_mask[mask]\n",
    "\n",
    "#df_train_mask.loc[mask, 'service_4'] = np.nan\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>service_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344887</th>\n",
       "      <td>1259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344888</th>\n",
       "      <td>604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344889</th>\n",
       "      <td>603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344890</th>\n",
       "      <td>603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344891</th>\n",
       "      <td>178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>344892 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        service_title\n",
       "0                 907\n",
       "1                1259\n",
       "2                 263\n",
       "3                 107\n",
       "4                 234\n",
       "...               ...\n",
       "344887           1259\n",
       "344888            604\n",
       "344889            603\n",
       "344890            603\n",
       "344891            178\n",
       "\n",
       "[344892 rows x 1 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
       "       'external_status_1', 'order_type_1', 'department_id_1',\n",
       "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
       "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
       "       'sole_1', 'legal_1', 'auto_ping_queue_1', 'requester_type', 'gender'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'order_type_2', 'department_id_2',\n",
    "       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',\n",
    "       'legal_2', 'auto_ping_queue_2','service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'order_type_3', 'department_id_3', 'custom_service_id_3',\n",
    "       'service_level_3', 'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', 'day_part_3', 'person_3', 'sole_3', 'legal_3',\n",
    "       'auto_ping_queue_3',\n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1',\n",
    "       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',\n",
    "       'external_status_2', 'department_id_2', 'month_2', 'week_2', 'year_2',\n",
    "       'custom_service_id_2', 'is_subdep_2', 'is_csid_2',\n",
    "       'dayofweek_2', 'service_3',\n",
    "       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',\n",
    "       'department_id_3', 'custom_service_id_3',\n",
    "       'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',\n",
    "       'dayofweek_3', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train\n",
    "y = df_test['service_title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1', 'order_type_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'service_level_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',\n",
    "       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',\n",
    "       'requester_type', 'gender']\n",
    "\n",
    "cat = ['service_1', 'service_title_1', 'mfc_1',\n",
    "       'internal_status_1', 'external_status_1',\n",
    "       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',\n",
    "       'is_subdep_1', 'is_csid_1', 'dayofweek_1', \n",
    "       'requester_type', 'gender']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[cat] = X[cat].astype('Int64')\n",
    "X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X[cat] = X[cat].astype('Int64')\n",
    "#X[cat] = X[cat].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[categorical] = X[categorical].fillna('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df):\n",
    "    \"\"\" \n",
    "    iterate through all the columns of a dataframe and \n",
    "    modify the data type to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage of dataframe is {:.2f}' \n",
    "                     'MB').format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max <\\\n",
    "                  np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max <\\\n",
    "                   np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max <\\\n",
    "                   np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max <\\\n",
    "                   np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max <\\\n",
    "                   np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max <\\\n",
    "                   np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(('Memory usage after optimization is: {:.2f}' \n",
    "                              'MB').format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) \n",
    "                                             / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 57.89MB\n",
      "Memory usage after optimization is: 13.82MB\n",
      "Decreased by 76.1%\n"
     ]
    }
   ],
   "source": [
    "X = reduce_mem_usage(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X[categorical]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['person_1'] = X['person_1'].astype('int32')\n",
    "X['sole_1'] = X['sole_1'].astype('int32')\n",
    "X['legal_1'] = X['legal_1'].astype('int32')\n",
    "X['auto_ping_queue_1'] = X['auto_ping_queue_1'].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "labeling = []\n",
    "for col in X.columns:\n",
    "    d = pd.DataFrame()\n",
    "    d[col] = X[col].unique()\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.fit(d[col])\n",
    "    d[col+'_l'] = le.transform(d[col])\n",
    "    d = d.sort_values(by=[col+'_l']).reset_index(drop=True)\n",
    "    labeling.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame()\n",
    "d['service_title'] = y.unique()\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(d['service_title'])\n",
    "d['service_title'+'_l'] = le.transform(d['service_title'])\n",
    "d = d.sort_values(by=['service_title'+'_l']).reset_index(drop=True)\n",
    "labeling_y = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>service_1</th>\n",
       "      <th>service_1_l</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2266851</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2266858</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2266865</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2266872</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46880280</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64279142</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64279302</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>64279389</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64279525</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>64279644</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64279761</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>65129676</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>81603166</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>97794844</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>113491165</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>113491201</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    service_1  service_1_l\n",
       "0          43            0\n",
       "1     2266851            1\n",
       "2     2266858            2\n",
       "3     2266865            3\n",
       "4     2266872            4\n",
       "5    46880280            5\n",
       "6    64279142            6\n",
       "7    64279302            7\n",
       "8    64279389            8\n",
       "9    64279525            9\n",
       "10   64279644           10\n",
       "11   64279761           11\n",
       "12   65129676           12\n",
       "13   81603166           13\n",
       "14   97794844           14\n",
       "15  113491165           15\n",
       "16  113491201           16"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labeling[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "for col in X.columns:\n",
    "    X[col] = X[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>service_1</th>\n",
       "      <th>service_title_1</th>\n",
       "      <th>mfc_1</th>\n",
       "      <th>internal_status_1</th>\n",
       "      <th>external_status_1</th>\n",
       "      <th>order_type_1</th>\n",
       "      <th>department_id_1</th>\n",
       "      <th>custom_service_id_1</th>\n",
       "      <th>service_level_1</th>\n",
       "      <th>is_subdep_1</th>\n",
       "      <th>...</th>\n",
       "      <th>day_part_1</th>\n",
       "      <th>month_1</th>\n",
       "      <th>week_1</th>\n",
       "      <th>year_1</th>\n",
       "      <th>person_1</th>\n",
       "      <th>sole_1</th>\n",
       "      <th>legal_1</th>\n",
       "      <th>auto_ping_queue_1</th>\n",
       "      <th>requester_type</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>757</td>\n",
       "      <td>127</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>541</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>63</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>156</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>68</td>\n",
       "      <td>33</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>62</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>201</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344887</th>\n",
       "      <td>3</td>\n",
       "      <td>545</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344888</th>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "      <td>138</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344889</th>\n",
       "      <td>0</td>\n",
       "      <td>358</td>\n",
       "      <td>140</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>63</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344890</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>133</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>63</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344891</th>\n",
       "      <td>0</td>\n",
       "      <td>101</td>\n",
       "      <td>124</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>344892 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        service_1  service_title_1  mfc_1  internal_status_1  \\\n",
       "0               4              757    127                  7   \n",
       "1              13              541      1                  7   \n",
       "2               0              156      2                  7   \n",
       "3               0               12     62                  7   \n",
       "4               0               73    201                  7   \n",
       "...           ...              ...    ...                ...   \n",
       "344887          3              545      4                  7   \n",
       "344888          0              239    138                  7   \n",
       "344889          0              358    140                  7   \n",
       "344890          0              137    133                  7   \n",
       "344891          0              101    124                  7   \n",
       "\n",
       "        external_status_1 order_type_1  department_id_1  custom_service_id_1  \\\n",
       "0                       3            3               15                    0   \n",
       "1                      17            3               63                    0   \n",
       "2                      17            3               68                   33   \n",
       "3                      17            3               16                    8   \n",
       "4                      17            3                8                   28   \n",
       "...                   ...          ...              ...                  ...   \n",
       "344887                  3            3               15                    0   \n",
       "344888                 17            3               26                    1   \n",
       "344889                 17            3               63                   25   \n",
       "344890                 17            3               63                   17   \n",
       "344891                 17            3                6                   40   \n",
       "\n",
       "       service_level_1  is_subdep_1  ...  day_part_1  month_1 week_1  year_1  \\\n",
       "0                    0            0  ...           1        0      4       1   \n",
       "1                    0            0  ...           1        3     15       1   \n",
       "2                    3            0  ...           1        3     15       0   \n",
       "3                    4            0  ...           1        7     31       0   \n",
       "4                    0            0  ...           0        1      4       0   \n",
       "...                ...          ...  ...         ...      ...    ...     ...   \n",
       "344887               0            0  ...           1        1      6       0   \n",
       "344888               1            0  ...           2        5     24       0   \n",
       "344889               0            0  ...           1        4     21       0   \n",
       "344890               0            0  ...           0        1      6       0   \n",
       "344891               0            0  ...           1        2     12       0   \n",
       "\n",
       "        person_1  sole_1  legal_1  auto_ping_queue_1  requester_type  gender  \n",
       "0              1       0        0                  0               0       1  \n",
       "1              1       0        0                  1               0       0  \n",
       "2              0       0        0                  0               0       0  \n",
       "3              0       0        0                  1               0       1  \n",
       "4              0       0        0                  1               0       1  \n",
       "...          ...     ...      ...                ...             ...     ...  \n",
       "344887         1       0        0                  1               0       0  \n",
       "344888         0       0        0                  0               0       1  \n",
       "344889         0       0        0                  1               0       1  \n",
       "344890         0       0        0                  1               0       1  \n",
       "344891         0       0        0                  1               0       0  \n",
       "\n",
       "[344892 rows x 22 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = y.map(labeling_y.set_index('service_title').to_dict()['service_title'+'_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "service_1 17\n",
      "service_title_1 785\n",
      "mfc_1 212\n",
      "internal_status_1 9\n",
      "external_status_1 23\n",
      "order_type_1 4\n",
      "department_id_1 78\n",
      "custom_service_id_1 51\n",
      "service_level_1 5\n",
      "is_subdep_1 2\n",
      "is_csid_1 2\n",
      "dayofweek_1 7\n",
      "day_part_1 4\n",
      "month_1 12\n",
      "week_1 52\n",
      "year_1 2\n",
      "person_1 2\n",
      "sole_1 2\n",
      "legal_1 2\n",
      "auto_ping_queue_1 2\n",
      "requester_type 3\n",
      "gender 2\n"
     ]
    }
   ],
   "source": [
    "for col in X.columns:\n",
    "    print(col, len(X[col].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "REPLICAS:  1\n"
     ]
    }
   ],
   "source": [
    "# Detect hardware, return appropriate distribution strategy\n",
    "try:\n",
    "    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n",
    "    # set: this is always the case on Kaggle.\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    tpu = None\n",
    "\n",
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "BATCH_SIZE = 32 * strategy.num_replicas_in_sync\n",
    "LEARNING_RATE = 1e-3 * strategy.num_replicas_in_sync \n",
    "EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    service_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_1')\n",
    "    service_title_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_1')\n",
    "    mfc_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_1')\n",
    "    internal_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_1')\n",
    "    external_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_1')\n",
    "    order_type_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_1')\n",
    "    department_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_1')\n",
    "    custom_service_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_1')\n",
    "    service_level_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_1')\n",
    "    is_subdep_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_1')\n",
    "    is_csid_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_1')\n",
    "    dayofweek_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_1')\n",
    "    day_part_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_1')\n",
    "    month_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_1')\n",
    "    week_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_1')\n",
    "    year_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_1')\n",
    "    person_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_1')\n",
    "    sole_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_1')\n",
    "    legal_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_1')\n",
    "    auto_ping_queue_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_1')\n",
    "    requester_type = tf.keras.layers.Input((1,), dtype=tf.int32, name='requester_type')\n",
    "    gender = tf.keras.layers.Input((1,), dtype=tf.int32, name='gender')\n",
    "    \n",
    "    service_1_embedding = tf.keras.layers.Embedding(len(X['service_1'].unique()), 11, input_length=1, name='service_1_embedding')(service_1)\n",
    "    service_title_1_embedding = tf.keras.layers.Embedding(len(X['service_title_1'].unique()), 50, input_length=1, name='service_title_1_embedding')(service_title_1)\n",
    "    mfc_1_embedding = tf.keras.layers.Embedding(len(X['mfc_1'].unique()), 50, input_length=1, name='mfc_1_embedding')(mfc_1)\n",
    "    internal_status_1_embedding = tf.keras.layers.Embedding(len(X['internal_status_1'].unique()), 5, input_length=1, name='internal_status_1_embedding')(internal_status_1)\n",
    "    external_status_1_embedding = tf.keras.layers.Embedding(len(X['external_status_1'].unique()), 14, input_length=1, name='external_status_1_embedding')(external_status_1)\n",
    "    order_type_1_embedding = tf.keras.layers.Embedding(len(X['order_type_1'].unique()), 2, input_length=1, name='order_type_1_embedding')(order_type_1)\n",
    "    department_id_1_embedding = tf.keras.layers.Embedding(len(X['department_id_1'].unique()), 46, input_length=1, name='department_id_1_embedding')(department_id_1)\n",
    "    custom_service_id_1_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_1'].unique()), 26, input_length=1, name='custom_service_id_1_embedding')(custom_service_id_1)\n",
    "    service_level_1_embedding = tf.keras.layers.Embedding(len(X['service_level_1'].unique()), 3, input_length=1, name='service_level_1_embedding')(service_level_1)\n",
    "    is_subdep_1_embedding = tf.keras.layers.Embedding(len(X['is_subdep_1'].unique()), 1, input_length=1, name='is_subdep_1_embedding')(is_subdep_1)\n",
    "    is_csid_1_embedding = tf.keras.layers.Embedding(len(X['is_csid_1'].unique()), 1, input_length=1, name='is_csid_1_embedding')(is_csid_1)\n",
    "    dayofweek_1_embedding = tf.keras.layers.Embedding(len(X['dayofweek_1'].unique()), 4, input_length=1, name='dayofweek_1_embedding')(dayofweek_1)\n",
    "    day_part_1_embedding = tf.keras.layers.Embedding(len(X['day_part_1'].unique()), 2, input_length=1, name='day_part_1_embedding')(day_part_1)\n",
    "    month_1_embedding = tf.keras.layers.Embedding(len(X['month_1'].unique()), 6, input_length=1, name='month_1_embedding')(month_1)\n",
    "    week_1_embedding = tf.keras.layers.Embedding(len(X['week_1'].unique()), 26, input_length=1, name='week_1_embedding')(week_1)\n",
    "    year_1_embedding = tf.keras.layers.Embedding(len(X['year_1'].unique()), 1, input_length=1, name='year_1_embedding')(year_1)\n",
    "    person_1_embedding = tf.keras.layers.Embedding(len(X['person_1'].unique()), 1, input_length=1, name='person_1_embedding')(person_1)\n",
    "    sole_1_embedding = tf.keras.layers.Embedding(len(X['sole_1'].unique()), 1, input_length=1, name='sole_1_embedding')(sole_1)\n",
    "    legal_1_embedding = tf.keras.layers.Embedding(len(X['legal_1'].unique()), 1, input_length=1, name='legal_1_embedding')(legal_1)\n",
    "    auto_ping_queue_1_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_1'].unique()), 1, input_length=1, name='auto_ping_queue_1_embedding')(auto_ping_queue_1)\n",
    "    requester_type_embedding = tf.keras.layers.Embedding(len(X['requester_type'].unique()), 2, input_length=1, name='requester_type_embedding')(requester_type)\n",
    "    gender_embedding = tf.keras.layers.Embedding(len(X['gender'].unique()), 1, input_length=1, name='gender_embedding')(gender)\n",
    "\n",
    "    concatenated = tf.keras.layers.Concatenate()([service_1_embedding, \n",
    "                                                  service_title_1_embedding,\n",
    "                                                 mfc_1_embedding,\n",
    "                                                 internal_status_1_embedding,\n",
    "                                                 external_status_1_embedding,\n",
    "                                                 order_type_1_embedding,\n",
    "                                                 department_id_1_embedding,\n",
    "                                                 custom_service_id_1_embedding,\n",
    "                                                 service_level_1_embedding,\n",
    "                                                 is_subdep_1_embedding,\n",
    "                                                 is_csid_1_embedding,\n",
    "                                                 dayofweek_1_embedding,\n",
    "                                                 day_part_1_embedding,\n",
    "                                                 month_1_embedding,\n",
    "                                                 week_1_embedding,\n",
    "                                                 year_1_embedding,\n",
    "                                                 person_1_embedding,\n",
    "                                                 sole_1_embedding,\n",
    "                                                 legal_1_embedding,\n",
    "                                                 auto_ping_queue_1_embedding,\n",
    "                                                 requester_type_embedding,\n",
    "                                                 gender_embedding])\n",
    "    #out = tf.keras.layers.Flatten()(concatenated)\n",
    "    \n",
    "    #out = tf.keras.layers.Dense(512, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Dense(256, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Dense(256, activation='relu')(out)\n",
    "    #out = tf.keras.layers.Conv1D(128, 2, padding='same')(concatenated)\n",
    "    #out = tf.keras.layers.LeakyReLU()(out)\n",
    "    #out = tf.keras.layers.Conv1D(64, 2, padding='same')(out)\n",
    "    #out = tf.keras.layers.Flatten()(out)\n",
    "    conv_0 = tf.keras.layers.Conv1D(128, 3, padding='same', activation='relu')(concatenated)\n",
    "    conv_1 = tf.keras.layers.Conv1D(128, 2, padding='same', activation='relu')(concatenated)\n",
    "    conv_2 = tf.keras.layers.Conv1D(128, 6, padding='same', activation='relu')(concatenated)\n",
    "    conv_0 = tf.keras.layers.Conv1D(64, 3, padding='same', activation='relu')(conv_0)\n",
    "    conv_1 = tf.keras.layers.Conv1D(64, 2, padding='same', activation='relu')(conv_1)\n",
    "    conv_2 = tf.keras.layers.Conv1D(64, 6, padding='same', activation='relu')(conv_2)\n",
    "    concatenated_tensor = tf.keras.layers.Concatenate(axis=1)([conv_0, conv_1, conv_2])\n",
    "    out = tf.keras.layers.Flatten()(concatenated_tensor)\n",
    "    \n",
    "    out = tf.keras.layers.Dropout(0.1)(out)\n",
    "    \n",
    "    out = tf.keras.layers.Dense(len(y.unique()), activation='softmax', name='service_title')(out)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=[service_1, \n",
    "                                          service_title_1, \n",
    "                                          mfc_1,\n",
    "                                         internal_status_1,\n",
    "                                         external_status_1,\n",
    "                                         order_type_1,\n",
    "                                         department_id_1,\n",
    "                                         custom_service_id_1,\n",
    "                                         service_level_1,\n",
    "                                         is_subdep_1,\n",
    "                                         is_csid_1,\n",
    "                                         dayofweek_1,\n",
    "                                         day_part_1,\n",
    "                                         month_1,\n",
    "                                         week_1,\n",
    "                                         year_1,\n",
    "                                         person_1,\n",
    "                                         sole_1,\n",
    "                                         legal_1,\n",
    "                                         auto_ping_queue_1,\n",
    "                                         requester_type,\n",
    "                                         gender], \n",
    "                                  outputs=out)\n",
    "\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model()\n",
    "model.load_weights('../input/emb-weights/v9-learnedemb-0.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
       "       'external_status_1', 'order_type_1', 'department_id_1',\n",
       "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
       "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
       "       'sole_1', 'legal_1', 'auto_ping_queue_1', 'requester_type', 'gender'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "embeddings = []\n",
    "j = 0\n",
    "for col in X.columns:\n",
    "    weights = model.get_layer(col + '_embedding').get_weights()[0].transpose()\n",
    "    d = labeling[j][:]\n",
    "    for i in range(weights.shape[0]):\n",
    "           d[col + '_e_' + str(i)] = weights[i]\n",
    "    j += 1\n",
    "    embeddings.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_emb = X_clustered[:]\n",
    "i = 0\n",
    "for col in X.columns:\n",
    "    X_emb = pd.merge(X_emb, embeddings[i], how='left', on=col)\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_emb.drop(['index', 'service_1', 'service_title_1', 'mfc_1', 'internal_status_1',\n",
    "       'external_status_1', 'order_type_1', 'department_id_1',\n",
    "       'custom_service_id_1', 'service_level_1', 'is_subdep_1', 'is_csid_1',\n",
    "       'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1', 'person_1',\n",
    "       'sole_1', 'legal_1', 'auto_ping_queue_1', 'requester_type', 'gender', 'cluster'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_emb.drop(['service_1_l', 'service_title_1_l', 'mfc_1_l', 'internal_status_1_l',\n",
    "       'external_status_1_l', 'order_type_1_l', 'department_id_1_l',\n",
    "       'custom_service_id_1_l', 'service_level_1_l', 'is_subdep_1_l', 'is_csid_1_l',\n",
    "       'dayofweek_1_l', 'day_part_1_l', 'month_1_l', 'week_1_l', 'year_1_l', 'person_1_l',\n",
    "       'sole_1_l', 'legal_1_l', 'auto_ping_queue_1_l', 'requester_type_l', 'gender_l'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_emb.to_csv('df_train_emb.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}