meli_challenge_make_busquedas_compras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import.\n",
    "import gzip\n",
    "import json\n",
    "import gc\n",
    "import math\n",
    "from collections import Counter, defaultdict\n",
    "from tqdm.notebook import tqdm\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from itertools import product\n",
    "from itertools import chain\n",
    "from itertools import compress\n",
    "from numba import jit, njit, vectorize\n",
    "import numpy_indexed as npi\n",
    "from ast import literal_eval\n",
    "\n",
    "def jl_to_list(fname):\n",
    "    output = []\n",
    "    with gzip.open(fname, 'rb') as f:\n",
    "        for line in f:\n",
    "            output.append(json.loads(line))\n",
    "    return output\n",
    "\n",
    "def cut_str(a):\n",
    "    try:\n",
    "        return a[4:]\n",
    "    except:\n",
    "        return a\n",
    "\n",
    "#Funciones.\n",
    "def dominios_visitados(visits):\n",
    "    domains = Counter()\n",
    "    for item in visits:\n",
    "        domain = metadata[item]['domain_code']\n",
    "        domains[domain] += 1\n",
    "    return domains\n",
    "\n",
    "def productos_visitados(visits):\n",
    "    productos = Counter()\n",
    "    for item in visits:\n",
    "        producto = metadata[item]['product_id']\n",
    "        if producto:\n",
    "            productos[producto] += 1\n",
    "    return productos\n",
    "\n",
    "def categorias_visitadas(visits):\n",
    "    categorias = Counter()\n",
    "    for item in visits:\n",
    "        categoria = metadata[item]['category_code']\n",
    "        if categoria:\n",
    "            categorias[categoria] += 1\n",
    "    return categorias\n",
    "\n",
    "def top_items(feature, k=10, m='d'):\n",
    "    vector = {'d': ventas_x_dominio,\n",
    "          'p': ventas_x_producto,\n",
    "          'c': ventas_x_categoria}[m]\n",
    "    top = vector[feature]\n",
    "    top = Counter(top)\n",
    "    top = top.most_common(k)\n",
    "    return [x[0] for x in top]\n",
    "\n",
    "def last_viewed(row, k=10):\n",
    "    a = []\n",
    "    for i in sorted([ev for ev in row if ev['event_type']=='view'], key = lambda x: x['event_timestamp'], reverse = True):\n",
    "        if i['event_info'] not in a:\n",
    "            a.append(i['event_info'])\n",
    "        if len(a) == k:\n",
    "            break\n",
    "    j = k - len(a)\n",
    "    a.extend([0] * j)\n",
    "    return a\n",
    "\n",
    "def most_viewed(row, k=10):\n",
    "    viewed = views(row)\n",
    "    m_viewed = Counter()\n",
    "    if len(viewed)==0:\n",
    "        return [0] * k\n",
    "    for item in viewed:\n",
    "        m_viewed[item] += 1\n",
    "    j = min(len(m_viewed), k)\n",
    "    a = [x[0] for x in m_viewed.most_common(j)]\n",
    "    a.extend([0] * (k - j))\n",
    "    return a\n",
    "\n",
    "def top_by_best_domain(row, k=10):\n",
    "    viewed = views(row)\n",
    "    if len(viewed)==0:\n",
    "        return [0] * k\n",
    "    domain = dominios_visitados(viewed)\n",
    "    domain = domain.most_common(1)[0][0]\n",
    "    t_items = top_items(domain, k=k, m='d')\n",
    "    if len(t_items) < k:\n",
    "        t_items.extend([0] * (k - len(t_items)))\n",
    "    return t_items\n",
    "\n",
    "def top_by_best_product(row, k=10):\n",
    "    viewed = views(row)\n",
    "    if len(viewed)==0:\n",
    "        return [0] * k\n",
    "    producto = productos_visitados(viewed)\n",
    "    try:\n",
    "        producto = producto.most_common(1)[0][0]\n",
    "        t_items = top_items(producto, k=k, m='p')\n",
    "        if len(t_items) < k:\n",
    "            t_items.extend([0] * (k - len(t_items)))\n",
    "        return t_items\n",
    "    except:\n",
    "        return [0] * k\n",
    "\n",
    "def top_by_best_category(row, k=10):\n",
    "    viewed = views(row)\n",
    "    if len(viewed)==0:\n",
    "        return [0] * k\n",
    "    categoria = categorias_visitadas(viewed)\n",
    "    try:\n",
    "        categoria = categoria.most_common(1)[0][0]\n",
    "    except:\n",
    "        return [0] * k\n",
    "    t_items = top_items(categoria, k=k, m='c')\n",
    "    if len(t_items) < k:\n",
    "        t_items.extend([0] * (k - len(t_items)))\n",
    "    return t_items\n",
    "\n",
    "def top_site(row, k=10):\n",
    "    return [x[0] for x in ventas.most_common(k)]\n",
    "\n",
    "def get_item_bought(row):\n",
    "    if 'item_bought' in row:\n",
    "        return row['item_bought']\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def ib_same_as_liv(ib, liv):\n",
    "    return ib==liv\n",
    "\n",
    "def ib_in_viewed(row, viewed):\n",
    "    ib = get_item_bought(row)\n",
    "    return ib in viewed\n",
    "\n",
    "def history_length(row):\n",
    "    return len(row['user_history'])\n",
    "\n",
    "def views0(row):\n",
    "    return ([ev['event_info'] for ev in row['user_history'] if ev['event_type']=='view'])\n",
    "\n",
    "def views(row):\n",
    "    return ([ev['event_info'] for ev in row if ev['event_type']=='view'])\n",
    "\n",
    "def searchs(row):\n",
    "    return ([ev['event_info'] for ev in row['user_history'] if ev['event_type']=='search'])\n",
    "\n",
    "def ib_same_as_miv(ib, miv):\n",
    "    return ib==miv\n",
    "\n",
    "def make_target_list(rows_train, extract_function, k=10):\n",
    "    target = []\n",
    "    for row in tqdm(rows_train):\n",
    "        target.append([extract_function(row)] * k)\n",
    "    return target\n",
    "\n",
    "def make_features_dict_list(rows_train, features_def, aux_function, k=10):\n",
    "    features = []\n",
    "    for row in tqdm(rows_train):\n",
    "        aux_data = aux_function(row)\n",
    "        row_features = {}\n",
    "        for f in features_def:\n",
    "            row_features[f] = features_def[f](row, aux_data, k=k)\n",
    "        features.append(row_features)\n",
    "    return features\n",
    "\n",
    "def make_features_pd(rows_train, features_def, k=10):\n",
    "    ROWS_TRAIN = pd.DataFrame(rows_train)\n",
    "    for f in tqdm(features_def):\n",
    "        ROWS_TRAIN[f] = ROWS_TRAIN['user_history'].apply(features_def[f])\n",
    "    return ROWS_TRAIN\n",
    "    \n",
    "def list_to_np_array(collection):\n",
    "    if str(type(collection[0])).replace(\"'\",\"\") == '<class list>':\n",
    "        for i, j in tqdm(enumerate(collection)):\n",
    "            collection[i] = np.array(j)\n",
    "    elif str(type(collection[0])).replace(\"'\",\"\") == '<class dict>':\n",
    "        for i, j in tqdm(enumerate(collection)):\n",
    "            for h in collection[i]:\n",
    "                collection[i][h] = np.array(collection[i][h])\n",
    "    return collection\n",
    "\n",
    "def searchs_to_np(searchs):\n",
    "    return np.array([busquedas_values[x] for x in searchs])\n",
    "\n",
    "def add_features_to_list(new_features, new_feature_data, old_feature_data):\n",
    "    if len(new_feature_data) == len(old_feature_data):\n",
    "        for row in tqdm(zip(old_feature_data, new_feature_data)):\n",
    "            for key in new_features:\n",
    "                row[0][key] = row[1][key]\n",
    "    return old_feature_data\n",
    "\n",
    "def str_to_int(a):\n",
    "    return int(a)\n",
    "\n",
    "def extract_views(a):\n",
    "    b = np.empty_like(a)\n",
    "    for i in range(len(b)):\n",
    "        b[i] = np.array([x['event_info'] for x in a[i] if x['event_type'] == 'view'])\n",
    "    return b\n",
    "\n",
    "def extract_searchs(a):\n",
    "    b = np.empty_like(a)\n",
    "    for i in range(len(b)):\n",
    "        b[i] = np.array([x['event_info'] for x in a[i] if x['event_type'] == 'search'])\n",
    "    return b\n",
    "\n",
    "def vistas_compras(a, b, c):\n",
    "    d = []\n",
    "    for i in np.unique(a):\n",
    "        if i != b:\n",
    "            d.append([i, b, c, 1])\n",
    "    return d\n",
    "\n",
    "def gen_vistas_compras(views, ib):\n",
    "    id = np.array(range(len(views)))\n",
    "    VC = []\n",
    "    for v in tqdm(range(len(views))):\n",
    "        vc = vistas_compras(views[v][0], ib[v][0], id[v])\n",
    "        VC += vc\n",
    "    return VC\n",
    "\n",
    "def busquedas_compras(a, b, c):\n",
    "    d = []\n",
    "    for i in np.unique(a):\n",
    "        d.append([i, b, c, 1])\n",
    "    return d\n",
    "\n",
    "def gen_busquedas_compras(searchs, ib):\n",
    "    id = np.array(range(len(searchs)))\n",
    "    VC = []\n",
    "    for v in tqdm(range(len(searchs))):\n",
    "        vc = vistas_compras(searchs[v][0], ib[v][0], id[v])\n",
    "        VC += vc\n",
    "    return VC\n",
    "\n",
    "#Filtros nb.\n",
    "def filter_not(x, z):\n",
    "    return x != z\n",
    "filter_not_v = np.vectorize(filter_not)\n",
    "\n",
    "def filter_in(x, z):\n",
    "    return np.isin(x, z, assume_unique=True)\n",
    "filter_in_v = np.vectorize(filter_in)\n",
    "\n",
    "#Generar datos.\n",
    "def gen_data(window, events, k=10):\n",
    "    WINDOW = pd.DataFrame(window, columns=['busqueda', 'compra', 'id_sesion', 'constante'])\n",
    "    pred = []\n",
    "    j = 0\n",
    "    for b in tqdm(events):\n",
    "        h = list(WINDOW.loc[(WINDOW['id_sesion'] != j) & (np.isin(np.array(WINDOW['busqueda']), b, assume_unique=True))].groupby('compra').sum().sort_values('constante', ascending=False).index[:k])\n",
    "        if len(h) < k:\n",
    "            h.extend([-1] * (k - len(h)))\n",
    "        pred.append(h)\n",
    "        j += 1\n",
    "        if j % 1000 == 0:\n",
    "            PRED = pd.DataFrame(pred, columns=['pred'+str(i) for i in range(10)])\n",
    "            PRED.to_csv('dim_bc'+str(j)+'.csv', index=False, sep=';')        \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ib():\n",
    "    name = 'ib'\n",
    "    V = pd.read_csv(name+'.csv', sep=';')\n",
    "    return np.array(V)\n",
    "\n",
    "def load_views(uq, test):\n",
    "    name = 'views' + {True: '_test', False: '_train'}[test] + {True: '_uq', False: ''}[uq]\n",
    "    V = pd.read_csv(name+'.csv', sep=';')\n",
    "    V[name] = V[name].apply(lambda x: literal_eval(x))\n",
    "    V[name] = V[name].apply(lambda x: np.array(x))\n",
    "    return np.array(V)\n",
    "\n",
    "def load_searchs(uq, test):\n",
    "    name = 'searchs' + {True: '_test', False: '_train'}[test] + {True: '_uq', False: ''}[uq]\n",
    "    V = pd.read_csv(name+'.csv', sep=';')\n",
    "    V[name] = V[name].apply(lambda x: literal_eval(x))\n",
    "    V[name] = V[name].apply(lambda x: np.array(x))\n",
    "    return np.array(V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading data.\n",
    "ib = load_ib()\n",
    "searchs_train = load_searchs(uq=True, test=False)\n",
    "searchs_test = load_searchs(uq=True, test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "997b49214c074e5987866ed0fb989f69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=413163), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "BC = gen_busquedas_compras(searchs_train, ib)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dac1855346b5449faede3ccf242e589b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=177070), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py:569: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#dim_bc = gen_data(BC, searchs_train)\n",
    "dim_pred_bc = gen_data(BC, searchs_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dim_bc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-41f8936151c9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mDIM_BC\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdim_bc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'pred'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mDIM_BC\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dim_bc.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m';'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'dim_bc' is not defined"
     ]
    }
   ],
   "source": [
    "DIM_BC = pd.DataFrame(dim_bc, columns=['pred'+str(i) for i in range(10)])\n",
    "DIM_BC.to_csv('dim_bc.csv', index=False, sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM_PRED_BC = pd.DataFrame(dim_pred_bc, columns=['pred'+str(i) for i in range(10)])\n",
    "DIM_PRED_BC.to_csv('dim_pred_bc.csv', index=False, sep=';')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}