Avito Challenge_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Avito Challenge _ v2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "notebookstart= time.time()\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import gc\n",
    "#print(\"Data:\\n\",os.listdir(\"../input\"))\n",
    "\n",
    "# Models Packages\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import feature_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "\n",
    "# Gradient Boosting\n",
    "import lightgbm as lgb\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.cross_validation import KFold\n",
    "\n",
    "# Tf-Idf\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from scipy.sparse import hstack, csr_matrix\n",
    "from nltk.corpus import stopwords \n",
    "\n",
    "# Viz\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import string\n",
    "\n",
    "NFOLDS = 5\n",
    "SEED = 42\n",
    "VALID = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SklearnWrapper(object):\n",
    "    def __init__(self, clf, seed=0, params=None, seed_bool = True):\n",
    "        if(seed_bool == True):\n",
    "            params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "        \n",
    "def get_oof(clf, x_train, y, x_test):\n",
    "    oof_train = np.zeros((ntrain,))\n",
    "    oof_test = np.zeros((ntest,))\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))\n",
    "\n",
    "    for i, (train_index, test_index) in enumerate(kf):\n",
    "        print('\\nFold {}'.format(i))\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict(x_te)\n",
    "        oof_test_skf[i, :] = clf.predict(x_test)\n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n",
    "    \n",
    "def cleanName(text):\n",
    "    try:\n",
    "        textProc = text.lower()\n",
    "        # textProc = \" \".join(map(str.strip, re.split('(\\d+)',textProc)))\n",
    "        #regex = re.compile(u'[^[:alpha:]]')\n",
    "        #textProc = regex.sub(\" \", textProc)\n",
    "        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)\n",
    "        textProc = \" \".join(textProc.split())\n",
    "        return textProc\n",
    "    except: \n",
    "        return \"name error\"\n",
    "    \n",
    "    \n",
    "def rmse(y, y0):\n",
    "    assert len(y) == len(y0)\n",
    "    return np.sqrt(np.mean(np.power((y - y0), 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Data Load Stage\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nData Load Stage\")\n",
    "training = pd.read_csv('E:/_Gen/Avito Competition/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\n",
    "traindex = training.index\n",
    "testing = pd.read_csv('E:/_Gen/Avito Competition/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\n",
    "testdex = testing.index\n",
    "\n",
    "ntrain = training.shape[0]\n",
    "ntest = testing.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: 1503424 Rows, 16 Columns\n",
      "Test shape: 508438 Rows, 16 Columns\n"
     ]
    }
   ],
   "source": [
    "kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)\n",
    "\n",
    "y = training.deal_probability.copy()\n",
    "training.drop(\"deal_probability\",axis=1, inplace=True)\n",
    "print('Train shape: {} Rows, {} Columns'.format(*training.shape))\n",
    "print('Test shape: {} Rows, {} Columns'.format(*testing.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Combine Train and Test\n",
      "\n",
      "All Data shape: 2011862 Rows, 16 Columns\n"
     ]
    }
   ],
   "source": [
    "print(\"Combine Train and Test\")\n",
    "df = pd.concat([training,testing],axis=0)\n",
    "del training, testing\n",
    "gc.collect()\n",
    "print('\\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Engineering\n",
      "\n",
      "Create Time Variables\n"
     ]
    }
   ],
   "source": [
    "print(\"Feature Engineering\")\n",
    "df[\"price\"] = np.log(df[\"price\"]+0.001)\n",
    "df[\"price\"].fillna(df.price.mean(),inplace=True)\n",
    "df[\"image_top_1\"].fillna(-999,inplace=True)\n",
    "print(\"\\nCreate Time Variables\")\n",
    "df[\"Weekday\"] = df['activation_date'].dt.weekday\n",
    "#df[\"Weekd of Year\"] = df['activation_date'].dt.week\n",
    "#df[\"Day of Month\"] = df['activation_date'].dt.day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Validation Index and Remove Dead Variables\n",
    "training_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index\n",
    "validation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index\n",
    "df.drop([\"activation_date\",\"image\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Encode Variables\n",
      "Encoding : ['user_id', 'region', 'city', 'parent_category_name', 'category_name', 'user_type', 'image_top_1', 'param_1', 'param_2', 'param_3']\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nEncode Variables\")\n",
    "categorical = [\"user_id\",\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\",\"param_1\",\"param_2\",\"param_3\"]\n",
    "print(\"Encoding :\",categorical)\n",
    "\n",
    "# Encoder:\n",
    "lbl = preprocessing.LabelEncoder()\n",
    "for col in categorical:\n",
    "    df[col].fillna('Unknown')\n",
    "    df[col] = lbl.fit_transform(df[col].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Text Features\n",
      "\n",
      "[TF-IDF] Term Frequency Inverse Document Frequency Stage\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nText Features\")\n",
    "\n",
    "# Feature Engineering \n",
    "\n",
    "# Meta Text Features\n",
    "textfeats = [\"description\", \"title\"]\n",
    "df['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n",
    "\n",
    "df['title'] = df['title'].apply(lambda x: cleanName(x))\n",
    "df[\"description\"]   = df[\"description\"].apply(lambda x: cleanName(x))\n",
    "\n",
    "for cols in textfeats:\n",
    "    df[cols] = df[cols].astype(str) \n",
    "    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA\n",
    "    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n",
    "    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n",
    "    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n",
    "    df[cols + '_num_letters'] = df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n",
    "    \n",
    "# Extra Feature Engineering\n",
    "df['title_desc_len_ratio'] = df['title_num_letters']/df['description_num_letters']\n",
    "\n",
    "print(\"\\n[TF-IDF] Term Frequency Inverse Document Frequency Stage\")\n",
    "russian_stop = set(stopwords.words('russian'))\n",
    "\n",
    "tfidf_para = {\n",
    "    \"stop_words\": russian_stop,\n",
    "    \"analyzer\": 'word',\n",
    "    \"token_pattern\": r'\\w{1,}',\n",
    "    \"sublinear_tf\": True,\n",
    "    \"dtype\": np.float32,\n",
    "    \"norm\": 'l2',\n",
    "    #\"min_df\":5,\n",
    "    #\"max_df\":.9,\n",
    "    \"smooth_idf\":False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_col(col_name): return lambda x: x[col_name]\n",
    "##I added to the max_features of the description. It did not change my score much but it may be worth investigating\n",
    "vectorizer = FeatureUnion([\n",
    "        ('description',TfidfVectorizer(\n",
    "            ngram_range=(1, 2),\n",
    "            max_features=17000,\n",
    "            **tfidf_para,\n",
    "            preprocessor=get_col('description'))),\n",
    "        ('title',CountVectorizer(\n",
    "            ngram_range=(1, 2),\n",
    "            stop_words = russian_stop,\n",
    "            #max_features=7000,\n",
    "            preprocessor=get_col('title')))\n",
    "    ])\n",
    "    \n",
    "start_vect=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-c17c8806eb2a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#Fit my vectorizer on the entire dataset instead of the training rows\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#Score improved by .0001\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'records'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mready_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'records'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    712\u001b[0m         transformers = Parallel(n_jobs=self.n_jobs)(\n\u001b[0;32m    713\u001b[0m             \u001b[0mdelayed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_fit_one_transformer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrans\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 714\u001b[1;33m             for _, trans, _ in self._iter())\n\u001b[0m\u001b[0;32m    715\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_update_transformer_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtransformers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    716\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    777\u001b[0m             \u001b[1;31m# was dispatched. In particular this covers the edge\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    778\u001b[0m             \u001b[1;31m# case of Parallel used with an exhausted iterator.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 779\u001b[1;33m             \u001b[1;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    780\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    781\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    623\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    624\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 625\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    626\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    586\u001b[0m         \u001b[0mdispatch_timestamp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    587\u001b[0m         \u001b[0mcb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBatchCompletionCallBack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdispatch_timestamp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 588\u001b[1;33m         \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    589\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    590\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    109\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    330\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 332\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    333\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36m_fit_one_transformer\u001b[1;34m(transformer, X, y)\u001b[0m\n\u001b[0;32m    565\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_fit_one_transformer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 567\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    568\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    569\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m   1359\u001b[0m         \u001b[0mself\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1360\u001b[0m         \"\"\"\n\u001b[1;32m-> 1361\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mTfidfVectorizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1362\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_tfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m    867\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    868\u001b[0m         vocabulary, X = self._count_vocab(raw_documents,\n\u001b[1;32m--> 869\u001b[1;33m                                           self.fixed_vocabulary_)\n\u001b[0m\u001b[0;32m    870\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    871\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py\u001b[0m in \u001b[0;36m_count_vocab\u001b[1;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[0;32m    801\u001b[0m                     \u001b[1;32mcontinue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    802\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 803\u001b[1;33m             \u001b[0mj_indices\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature_counter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    804\u001b[0m             \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature_counter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    805\u001b[0m             \u001b[0mindptr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mj_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Fit my vectorizer on the entire dataset instead of the training rows\n",
    "#Score improved by .0001\n",
    "vectorizer.fit(df.to_dict('records'))\n",
    "\n",
    "ready_df = vectorizer.transform(df.to_dict('records'))\n",
    "tfvocab = vectorizer.get_feature_names()\n",
    "print(\"Vectorization Runtime: %0.2f Minutes\"%((time.time() - start_vect)/60))\n",
    "\n",
    "# Drop Text Cols\n",
    "textfeats = [\"description\", \"title\"]\n",
    "df.drop(textfeats, axis=1,inplace=True)\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "ridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,\n",
    "                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Ridge oof method from Faron's kernel\n",
    "#I was using this to analyze my vectorization, but figured it would be interesting to add the results back into the dataset\n",
    "#It doesn't really add much to the score, but it does help lightgbm converge faster\n",
    "ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)\n",
    "ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])\n",
    "\n",
    "rms = sqrt(mean_squared_error(y, ridge_oof_train))\n",
    "print('Ridge OOF RMSE: {}'.format(rms))\n",
    "\n",
    "print(\"Modeling Stage\")\n",
    "\n",
    "ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])\n",
    "\n",
    "df['ridge_preds'] = ridge_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine Dense Features with Sparse Text Bag of Words Features\n",
    "X = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]]) # Sparse Matrix\n",
    "testing = hstack([csr_matrix(df.loc[testdex,:].values),ready_df[traindex.shape[0]:]])\n",
    "tfvocab = df.columns.tolist() + tfvocab\n",
    "for shape in [X,testing]:\n",
    "    print(\"{} Rows and {} Cols\".format(*shape.shape))\n",
    "print(\"Feature Names Length: \",len(tfvocab))\n",
    "del df\n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\nModeling Stage\")\n",
    "\n",
    "del ridge_preds,vectorizer,ready_df\n",
    "gc.collect();\n",
    "    \n",
    "print(\"Light Gradient Boosting Regressor\")\n",
    "lgbm_params =  {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'regression',\n",
    "    'metric': 'rmse',\n",
    "    # 'max_depth': 15,\n",
    "    'num_leaves': 270,\n",
    "    'feature_fraction': 0.5,\n",
    "    'bagging_fraction': 0.75,\n",
    "    'bagging_freq': 2,\n",
    "    'learning_rate': 0.016,\n",
    "    'verbose': 0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if VALID == True:\n",
    "    X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "        X, y, test_size=0.10, random_state=23)\n",
    "        \n",
    "    # LGBM Dataset Formatting \n",
    "    lgtrain = lgb.Dataset(X_train, y_train,\n",
    "                    feature_name=tfvocab,\n",
    "                    categorical_feature = categorical)\n",
    "    lgvalid = lgb.Dataset(X_valid, y_valid,\n",
    "                    feature_name=tfvocab,\n",
    "                    categorical_feature = categorical)\n",
    "    del X, X_train; gc.collect()\n",
    "    \n",
    "    # Go Go Go\n",
    "    lgb_clf = lgb.train(\n",
    "        lgbm_params,\n",
    "        lgtrain,\n",
    "        num_boost_round=20000,\n",
    "        valid_sets=[lgtrain, lgvalid],\n",
    "        valid_names=['train','valid'],\n",
    "        early_stopping_rounds=50,\n",
    "        verbose_eval=100\n",
    "    )\n",
    "    print(\"Model Evaluation Stage\")\n",
    "    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, lgb_clf.predict(X_valid))))\n",
    "    del X_valid ; gc.collect()\n",
    "\n",
    "else:\n",
    "    # LGBM Dataset Formatting \n",
    "    lgtrain = lgb.Dataset(X, y,\n",
    "                    feature_name=tfvocab,\n",
    "                    categorical_feature = categorical)\n",
    "    del X; gc.collect()\n",
    "    # Go Go Go\n",
    "    lgb_clf = lgb.train(\n",
    "        lgbm_params,\n",
    "        lgtrain,\n",
    "        num_boost_round=1550,\n",
    "        verbose_eval=100\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature Importance Plot\n",
    "f, ax = plt.subplots(figsize=[7,10])\n",
    "lgb.plot_importance(lgb_clf, max_num_features=75, ax=ax)\n",
    "plt.title(\"Light GBM Feature Importance\")\n",
    "plt.savefig('feature_import.png')\n",
    "\n",
    "print(\"Model Evaluation Stage\")\n",
    "lgpred = lgb_clf.predict(testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mixing lightgbm with ridge. I haven't really tested if this improves the score or not\n",
    "#blend = 0.95*lgpred + 0.05*ridge_oof_test[:,0]\n",
    "lgsub = pd.DataFrame(lgpred,columns=[\"deal_probability\"],index=testdex)\n",
    "lgsub['deal_probability'].clip(0.0, 1.0, inplace=True) # Between 0 and 1\n",
    "lgsub.to_csv(\"lgsub.csv\",index=True,header=True)\n",
    "#print(\"Model Runtime: %0.2f Minutes\"%((time.time() - modelstart)/60))\n",
    "print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}