Title_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import torch\n",
    "import transformers as ppb\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"Final_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Title</th>\n",
       "      <th>Post Text</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Testing requirements in toys and games category</td>\n",
       "      <td>My product is domino sets for adults. I am the...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Food and Drug Administration</td>\n",
       "      <td>Hello, I am on my way to becoming an Amazon de...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Images Storage for mass upload</td>\n",
       "      <td>When uploading products with the spreadsheet w...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Prepaid Return Label program email clarification</td>\n",
       "      <td>You may have received an email from Amazon inf...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Pulling Amazon Custom Personalizatoin Data fro...</td>\n",
       "      <td>The problem: Currently Amazon gives you links ...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                              Title  \\\n",
       "0           0    Testing requirements in toys and games category   \n",
       "1           1                       Food and Drug Administration   \n",
       "2           2                     Images Storage for mass upload   \n",
       "3           3   Prepaid Return Label program email clarification   \n",
       "4           4  Pulling Amazon Custom Personalizatoin Data fro...   \n",
       "\n",
       "                                           Post Text       Category  \n",
       "0  My product is domino sets for adults. I am the...  Amazon Custom  \n",
       "1  Hello, I am on my way to becoming an Amazon de...  Amazon Custom  \n",
       "2  When uploading products with the spreadsheet w...  Amazon Custom  \n",
       "3  You may have received an email from Amazon inf...  Amazon Custom  \n",
       "4  The problem: Currently Amazon gives you links ...  Amazon Custom  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Testing requirements in toys and games category</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Food and Drug Administration</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Images Storage for mass upload</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Prepaid Return Label program email clarification</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pulling Amazon Custom Personalizatoin Data fro...</td>\n",
       "      <td>Amazon Custom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title       Category\n",
       "0    Testing requirements in toys and games category  Amazon Custom\n",
       "1                       Food and Drug Administration  Amazon Custom\n",
       "2                     Images Storage for mass upload  Amazon Custom\n",
       "3   Prepaid Return Label program email clarification  Amazon Custom\n",
       "4  Pulling Amazon Custom Personalizatoin Data fro...  Amazon Custom"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subdata = df[[\"Title\", \"Category\"]]\n",
    "subdata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Category</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1657</th>\n",
       "      <td>Violation of Amazon Marketplace Fair Pricing P...</td>\n",
       "      <td>Account Health</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>Question : AMAZONE’s packing service</td>\n",
       "      <td>Fulfillment By Amazon</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Seller Support for Amazon Custom Sellers</td>\n",
       "      <td>Amazon Custom</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1757</th>\n",
       "      <td>Orders for Out of Stock product</td>\n",
       "      <td>Account Health</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1690</th>\n",
       "      <td>Suspended, need advice and clarification</td>\n",
       "      <td>Account Health</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Title  \\\n",
       "1657  Violation of Amazon Marketplace Fair Pricing P...   \n",
       "399                Question : AMAZONE’s packing service   \n",
       "71             Seller Support for Amazon Custom Sellers   \n",
       "1757                    Orders for Out of Stock product   \n",
       "1690           Suspended, need advice and clarification   \n",
       "\n",
       "                   Category  Label  \n",
       "1657         Account Health      0  \n",
       "399   Fulfillment By Amazon      2  \n",
       "71            Amazon Custom      1  \n",
       "1757         Account Health      0  \n",
       "1690         Account Health      0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subdata['Category'] = subdata['Category'].astype('category')\n",
    "subdata['Label'] = subdata['Category'].cat.codes.astype('int32')\n",
    "subdata.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_class, tokenizer_class, pretrained_weights = (ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')\n",
    "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n",
    "model = model_class.from_pretrained(pretrained_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized = subdata[\"Title\"].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       [101, 5604, 5918, 1999, 10899, 1998, 2399, 469...\n",
       "1                      [101, 2833, 1998, 4319, 3447, 102]\n",
       "2         [101, 4871, 5527, 2005, 3742, 2039, 11066, 102]\n",
       "3       [101, 17463, 14326, 2709, 3830, 2565, 10373, 1...\n",
       "4       [101, 4815, 9733, 7661, 3167, 21335, 3406, 237...\n",
       "                              ...                        \n",
       "1810    [101, 1037, 6731, 4070, 2029, 2322, 1003, 2013...\n",
       "1811       [101, 2342, 2393, 2007, 4070, 8636, 5574, 102]\n",
       "1812                  [101, 7775, 4031, 3343, 13302, 102]\n",
       "1813    [101, 1022, 2243, 2006, 2907, 1011, 14768, 612...\n",
       "1814                 [101, 4070, 26709, 6593, 21967, 102]\n",
       "Name: Title, Length: 1815, dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  101,  5604,  5918, ...,     0,     0,     0],\n",
       "       [  101,  2833,  1998, ...,     0,     0,     0],\n",
       "       [  101,  4871,  5527, ...,     0,     0,     0],\n",
       "       ...,\n",
       "       [  101,  7775,  4031, ...,     0,     0,     0],\n",
       "       [  101,  1022,  2243, ...,     0,     0,     0],\n",
       "       [  101,  4070, 26709, ...,     0,     0,     0]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_len = 0\n",
    "for i in tokenized.values:\n",
    "    if len(i) > max_len:\n",
    "        max_len = len(i)\n",
    "\n",
    "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])\n",
    "\n",
    "padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1815, 46)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(padded).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1815, 46)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask = np.where(padded != 0, 1, 0)\n",
    "attention_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor(padded).long()\n",
    "attention_mask = torch.tensor(attention_mask)\n",
    "\n",
    "with torch.no_grad():\n",
    "    last_hidden_states = model(input_ids, attention_mask=attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[-0.1835, -0.0325, -0.0348,  ..., -0.3988,  0.1470,  0.3601],\n",
       "          [ 0.4766,  0.0457, -0.1548,  ..., -0.4201,  0.3834, -0.1362],\n",
       "          [ 0.1143,  0.2136, -0.0381,  ..., -0.4316, -0.3119,  0.0553],\n",
       "          ...,\n",
       "          [ 0.0537, -0.0326,  0.1003,  ..., -0.1306,  0.0262,  0.1715],\n",
       "          [ 0.1237,  0.0145,  0.3692,  ..., -0.2551,  0.1161,  0.0741],\n",
       "          [ 0.2589,  0.0167,  0.1903,  ..., -0.2201, -0.0424,  0.0101]],\n",
       " \n",
       "         [[-0.2713, -0.0437, -0.6490,  ..., -0.2204,  0.2402,  0.4417],\n",
       "          [-0.0381,  0.9230, -0.4835,  ..., -0.2055,  0.3211, -0.7571],\n",
       "          [-0.3356,  0.3825, -0.5847,  ...,  0.3219,  0.2007,  0.2282],\n",
       "          ...,\n",
       "          [ 0.2153, -0.1145, -0.0715,  ...,  0.1220, -0.2260,  0.2741],\n",
       "          [ 0.2495, -0.1307, -0.0541,  ...,  0.0763, -0.2571,  0.3050],\n",
       "          [ 0.2494, -0.0476, -0.1152,  ...,  0.0678, -0.2254,  0.2886]],\n",
       " \n",
       "         [[-0.3204, -0.2543, -0.0767,  ..., -0.2676, -0.1091,  0.2475],\n",
       "          [ 0.1039,  0.0143,  0.2420,  ..., -0.0576,  0.2556, -0.0293],\n",
       "          [ 0.0361,  0.1700,  0.2833,  ..., -0.2183, -0.3633, -0.2984],\n",
       "          ...,\n",
       "          [-0.1515, -0.1319,  0.2017,  ..., -0.1464, -0.2325,  0.0830],\n",
       "          [-0.0276, -0.1639,  0.2521,  ..., -0.1595, -0.2058, -0.0047],\n",
       "          [ 0.0221, -0.1526,  0.2334,  ..., -0.1306, -0.2240, -0.0292]],\n",
       " \n",
       "         ...,\n",
       " \n",
       "         [[-0.1769,  0.0363, -0.4426,  ..., -0.2106, -0.0260,  0.2609],\n",
       "          [ 0.1992,  0.0612, -0.3620,  ..., -0.2986,  0.0107, -0.1601],\n",
       "          [ 0.2665, -0.2795,  0.0990,  ..., -0.3063, -0.1046, -0.2206],\n",
       "          ...,\n",
       "          [ 0.2213,  0.1799, -0.1498,  ..., -0.0082, -0.5782, -0.0442],\n",
       "          [ 0.2077,  0.1906, -0.2265,  ...,  0.0147, -0.6376, -0.0026],\n",
       "          [ 0.2056,  0.1989, -0.2272,  ...,  0.0155, -0.6450,  0.0022]],\n",
       " \n",
       "         [[-0.3084, -0.1964, -0.2914,  ..., -0.0266,  0.1448,  0.5966],\n",
       "          [ 0.1480, -0.1244,  0.0846,  ..., -0.0488,  0.4196,  0.7570],\n",
       "          [-0.5324, -0.3291,  0.1162,  ..., -0.1285,  0.0296, -0.1809],\n",
       "          ...,\n",
       "          [-0.2912, -0.0096,  0.1482,  ...,  0.0361,  0.0264,  0.2091],\n",
       "          [-0.3682,  0.2904,  0.2120,  ...,  0.1064, -0.0044,  0.1928],\n",
       "          [ 0.3028,  0.0613,  0.4243,  ...,  0.0237, -0.1957,  0.1958]],\n",
       " \n",
       "         [[-0.4691, -0.1641, -0.1566,  ..., -0.1843,  0.0110,  0.3498],\n",
       "          [-0.0284, -0.3061,  0.0887,  ..., -0.1229, -0.1086,  0.2731],\n",
       "          [ 0.0407, -0.2330,  0.2528,  ..., -0.4716,  0.0823, -0.0193],\n",
       "          ...,\n",
       "          [-0.0982, -0.2028,  0.0853,  ...,  0.0102, -0.1955,  0.1280],\n",
       "          [-0.0687, -0.1869, -0.0656,  ...,  0.0416, -0.2380,  0.1314],\n",
       "          [-0.0436, -0.1683, -0.0689,  ...,  0.0371, -0.2447,  0.1135]]]),)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.18352728, -0.03250957, -0.03482962, ..., -0.39881077,\n",
       "         0.1470425 ,  0.36006016],\n",
       "       [-0.27133703, -0.04370546, -0.6490413 , ..., -0.22036885,\n",
       "         0.24016523,  0.44169772],\n",
       "       [-0.3203987 , -0.2542809 , -0.07665624, ..., -0.26764527,\n",
       "        -0.10913481,  0.24754797],\n",
       "       ...,\n",
       "       [-0.176929  ,  0.03628302, -0.4426356 , ..., -0.21058069,\n",
       "        -0.02595328,  0.26093438],\n",
       "       [-0.30844954, -0.19637042, -0.2913593 , ..., -0.02660279,\n",
       "         0.14479697,  0.5965741 ],\n",
       "       [-0.4690847 , -0.164107  , -0.15658677, ..., -0.18426396,\n",
       "         0.01099475,  0.34982154]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = last_hidden_states[0][:,0,:].numpy()\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = last_hidden_states[0][:,0,:].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = subdata['Label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_clf = LogisticRegression()\n",
    "lr_clf.fit(train_features, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7731277533039648"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_clf.score(test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}