tfidf_4_feats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import sys\n",
    "import re\n",
    "import json\n",
    "import time\n",
    "import operator\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from scipy.sparse import csr_matrix,coo_matrix,csc_matrix\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "from operator import itemgetter\n",
    "import scipy as sp\n",
    "import scipy.sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Install sparse dot topn\n",
    "\n",
    "#!pip3 install git+https://github.com/ing-bank/sparse_dot_topn.git\n",
    "if sys.version_info[0] >= 3:\n",
    "    from sparse_dot_topn import sparse_dot_topn as ct\n",
    "    from sparse_dot_topn import sparse_dot_topn_threaded as ct_thread\n",
    "else:\n",
    "    import sparse_dot_topn as ct\n",
    "    import sparse_dot_topn_threaded as ct_thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def timeit(method):\n",
    "    \"\"\"\n",
    "    Standard Python decorator that measures the execution time of a method;\n",
    "    \"\"\"\n",
    "    def timed(*args, **kw):\n",
    "        start = time.time()\n",
    "        result = method(*args, **kw)\n",
    "        end = time.time()\n",
    "        \n",
    "        #print(f\"{method.__name__}:  {(end - start):.2f} s\")\n",
    "        return result\n",
    "    return timed\n",
    "\n",
    "\n",
    "def read_file(path: str, set_record_id_as_index: bool=False) -> pd.DataFrame:\n",
    "    return pd.read_csv(path, dtype=str, escapechar=\"\\\\\", index_col=\"record_id\" if set_record_id_as_index else None)\n",
    "\n",
    "\n",
    "\n",
    "def recall_at_k(resultTable : pd.DataFrame, trainingData: pd.DataFrame, testingData: pd.DataFrame) -> dict:\n",
    "    \"\"\"\n",
    "    Given a list of K predictions for each query, first retrieve the correct ID from the test data,\n",
    "    then look in the training data the percentage of records that have been successfully identified.\n",
    "    \n",
    "    For example, given query \"1234-M\", first retrieve the correct ID \"1234\" from the test data,\n",
    "    then obtain from the training data all records that refer to \"1234\", \n",
    "    and finally look how many of them we have found;\n",
    "    \"\"\"\n",
    "    \n",
    "    # Obtain all the predictions for each record in the test set;\n",
    "    perQueryRecords = resultTable.groupby(\"queried_record_id\")\n",
    "    \n",
    "    # Group training records by their LinkedID truth value;\n",
    "    groupedTrainingRecords = trainingData.groupby(\"linked_id\")\n",
    "\n",
    "    totalRecall = 0.0\n",
    "\n",
    "    allRecords = dict()\n",
    "    \n",
    "    start = time.time()\n",
    "    for i, (queriedRecordID, group) in enumerate(perQueryRecords):\n",
    "        #if i % 1000 == 0 and i > 0:\n",
    "            #print(f\"processed {i}/{len(perQueryRecords)} records, {100 * i / len(perQueryRecords):.2f}%\")\n",
    "            #print(f\"\\tcurrent recall: {(totalRecall / i):.2f}\")\n",
    "            #print(f\"\\ttime elapsed: {(time.time() - start):.2f} s\")\n",
    "        \n",
    "        try:\n",
    "            queriedLinkedID = testingData.loc[queriedRecordID, \"linked_id\"]\n",
    "        except IndexError:\n",
    "            raise IndexError(\"ID {queriedRecordID} not found in testing data!\")\n",
    "        \n",
    "        try:\n",
    "            allRelevantRecords = set(groupedTrainingRecords.get_group(queriedLinkedID).index.values)\n",
    "        except KeyError:\n",
    "            allRelevantRecords = set()\n",
    "        setPredictedRecords = set(group[\"predicted_record_id\"])\n",
    "        selectedRelevantRecords = setPredictedRecords.intersection(allRelevantRecords)\n",
    "        recall = 1\n",
    "        if (len(allRelevantRecords) > 0):\n",
    "            recall = len(selectedRelevantRecords) / len(allRelevantRecords)\n",
    "\n",
    "        totalRecall += recall\n",
    "        allRecords[queriedRecordID] = [queriedRecordID, recall, len(selectedRelevantRecords), len(allRelevantRecords)]\n",
    "    \n",
    "    # Store the results in a summary table;\n",
    "    result_table =  pd.DataFrame.from_dict(\n",
    "                        allRecords,\n",
    "                        orient='index',\n",
    "                        columns=[\"QueriedRecordID\", \"Recall@K\", \"SelectedRecords\", \"AllRelevantRecords\"]\n",
    "                    )\n",
    "    # Compute the filtered recall, which considers only queries with at least one relevant record in the training data;\n",
    "    queries_with_relevant_records = result_table[result_table[\"AllRelevantRecords\"] > 0]\n",
    "    filtered_recall = np.mean(queries_with_relevant_records[\"SelectedRecords\"] / queries_with_relevant_records[\"AllRelevantRecords\"])\n",
    "\n",
    "    return {\n",
    "            \"AverageRecall\" : totalRecall / len(perQueryRecords),\n",
    "            \"AverageFilteredRecall\": filtered_recall,\n",
    "            \"perQueryResult\" : result_table\n",
    "            }\n",
    "    \n",
    "def precision_at_k(resultTable : pd.DataFrame, trainingData: pd.DataFrame, testingData: pd.DataFrame) -> dict:\n",
    "    \"\"\"\n",
    "    Given a list of K predictions for each query, first retrieve the correct ID from the test data,\n",
    "    then look in the training data the percentage of records that are actually relevant;\n",
    "    \n",
    "    For example, given query \"1234-M\", first retrieve the correct ID \"1234\" from the test data,\n",
    "    then obtain from the training data all records that refer to \"1234\", \n",
    "    and finally look how many of the records we have found are actually referring to \"1234\"\n",
    "    \"\"\"\n",
    "    \n",
    "    # Obtain all the predictions for each record in the test set;\n",
    "    perQueryRecords = resultTable.groupby(\"queried_record_id\")\n",
    "    \n",
    "    # Group training records by their LinkedID truth value;\n",
    "    groupedTrainingRecords = trainingData.groupby(\"linked_id\")\n",
    "\n",
    "    totalPrecision = 0.0\n",
    "    numberOfPredictionsForRelevantRecords = 0\n",
    "\n",
    "    allRecords = dict()\n",
    "    \n",
    "    start = time.time()\n",
    "    for i, (queriedRecordID, group) in enumerate(perQueryRecords):\n",
    "        #if i % 1000 == 0 and i > 0:\n",
    "            #print(f\"processed {i}/{len(perQueryRecords)} records, {100 * i / len(perQueryRecords):.2f}%\")\n",
    "            #print(f\"\\tcurrent precision: {(totalPrecision / i):.2f}\")\n",
    "            #print(f\"\\ttime elapsed: {(time.time() - start):.2f} s\")\n",
    "        \n",
    "        try:\n",
    "            queriedLinkedID = testingData.loc[queriedRecordID, \"linked_id\"]\n",
    "        except IndexError:\n",
    "            raise IndexError(\"ID {queriedRecordID} not found in testing data!\")\n",
    "        \n",
    "        try:\n",
    "            allRelevantRecords = set(groupedTrainingRecords.get_group(queriedLinkedID).index.values)\n",
    "        except KeyError:\n",
    "            allRelevantRecords = set()\n",
    "        setPredictedRecords = set(group[\"predicted_record_id\"])\n",
    "        selectedRelevantRecords = setPredictedRecords.intersection(allRelevantRecords)\n",
    "        precision = 1\n",
    "        if (len(allRelevantRecords) > 0):\n",
    "            precision = len(selectedRelevantRecords) / len(setPredictedRecords)\n",
    "            numberOfPredictionsForRelevantRecords += len(setPredictedRecords)\n",
    "\n",
    "        totalPrecision += precision\n",
    "        allRecords[queriedRecordID] = [queriedRecordID, precision, len(selectedRelevantRecords), len(allRelevantRecords)]\n",
    "    \n",
    "    # Store the results in a summary table;\n",
    "    result_table =  pd.DataFrame.from_dict(\n",
    "                        allRecords,\n",
    "                        orient='index',\n",
    "                        columns=[\"QueriedRecordID\", \"Precision@K\", \"SelectedRecords\", \"AllRelevantRecords\"]\n",
    "                    )\n",
    "    # Compute the filtered recall, which considers only queries with at least one relevant record in the training data;\n",
    "    queries_with_relevant_records = result_table[result_table[\"AllRelevantRecords\"] > 0]\n",
    "    filtered_precision = np.mean(queries_with_relevant_records[\"SelectedRecords\"] / numberOfPredictionsForRelevantRecords)\n",
    "\n",
    "    return {\n",
    "            \"AveragePrecision\" : totalPrecision / len(perQueryRecords),\n",
    "            \"AverageFilteredPrecision\": filtered_precision,\n",
    "            \"perQueryResult\" : result_table\n",
    "            }   \n",
    "\n",
    "#%%\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "@timeit\n",
    "def prediction_dict_to_df(predictions: dict) -> pd.DataFrame:\n",
    "    # Turn the prediction dict into a series of tuples;\n",
    "    results = []\n",
    "    for query_id, pred_list in predictions.items():\n",
    "        for p in pred_list:\n",
    "            results += [[query_id, p]]\n",
    "    return pd.DataFrame(results, columns=[\"queried_record_id\", \"predicted_record_id\"])\n",
    "\n",
    "@timeit\n",
    "def prediction_dict_to_kaggle_df(predictions: dict) -> pd.DataFrame:\n",
    "    # Turn the prediction dict into a series of tuples;\n",
    "    results = []\n",
    "    for query_id, pred_list in predictions.items():\n",
    "        results += [[query_id, \" \".join(pred_list)]]\n",
    "    return pd.DataFrame(results, columns=[\"queried_record_id\", \"predicted_record_id\"])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def awesome_cossim_top(A, B, ntop, lower_bound=0,use_threads=True,n_jobs=14):\n",
    "    # force A and B as a CSR matrix.\n",
    "    # If they have already been CSR, there is no overhead\n",
    "    A = A.tocsr()\n",
    "    B = B.tocsr()\n",
    "    M, _ = A.shape\n",
    "    _, N = B.shape\n",
    "    \n",
    "    idx_dtype = np.int32\n",
    " \n",
    "    nnz_max = M*ntop\n",
    " \n",
    "    indptr = np.zeros(M+1, dtype=idx_dtype)\n",
    "    indices = np.zeros(nnz_max, dtype=idx_dtype)\n",
    "    data = np.zeros(nnz_max, dtype=A.dtype)\n",
    "    ct_thread.sparse_dot_topn_threaded(\n",
    "        M, N, np.asarray(A.indptr, dtype=idx_dtype),\n",
    "        np.asarray(A.indices, dtype=idx_dtype),\n",
    "        A.data,\n",
    "        np.asarray(B.indptr, dtype=idx_dtype),\n",
    "        np.asarray(B.indices, dtype=idx_dtype),\n",
    "        B.data,\n",
    "        ntop,\n",
    "        lower_bound,\n",
    "        indptr, indices, data, n_jobs)\n",
    "    return csr_matrix((data,indices,indptr),shape=(M,N))\n",
    "\n",
    "def make_matchdf(x,source,target):\n",
    "    ''' Build dataframe for result return '''\n",
    "    # CSR matrix -> COO matrix\n",
    "    cx = x.tocoo()\n",
    "\n",
    "    # COO matrix to list of tuples\n",
    "    match_list = []\n",
    "    for row,col,val in zip(cx.row, cx.col, cx.data):\n",
    "        match_list.append((row,source[row], col, target[col], val))\n",
    "\n",
    "    # List of tuples to dataframe\n",
    "    colnames = ['Row Idx', 'Title', 'Candidate Idx', 'Candidate Title', 'Score']\n",
    "    match_df = pd.DataFrame(match_list, columns=colnames)\n",
    "\n",
    "    return match_df\n",
    "\n",
    "#Function useful to divide the score matrix by the weight matrix\n",
    "def sparse_divide_nonzero(a, b):\n",
    "    inv_b = b.copy()\n",
    "    inv_b.data = 1 / inv_b.data\n",
    "    return a.multiply(inv_b)\n",
    "\n",
    "\n",
    "def top_n_idx_sparse(matrix, n):\n",
    "    '''Return index of top n values in each row of a sparse matrix'''\n",
    "    top_n_idx = []\n",
    "    for le, ri in zip(matrix.indptr[:-1], matrix.indptr[1:]):\n",
    "        n_row_pick = min(n, ri - le)\n",
    "        top_n_idx.append(matrix.indices[le + np.argpartition(matrix.data[le:ri], -n_row_pick)[-n_row_pick:]])\n",
    "    return top_n_idx\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Loading dataset\n",
    "train=pd.read_csv('../panama-papers-polimi/data/panama_train_expanded_2.csv')\n",
    "test=pd.read_csv('../panama-papers-polimi/data/panama_test_expanded_2.csv')\n",
    "\n",
    "\n",
    "training_file = \"../panama-papers-polimi/data/entity-resolution_advanced-topics-training_data.csv\"\n",
    "train_or = read_file(training_file, set_record_id_as_index=False)\n",
    "        \n",
    "#X_train = train.drop(columns=\"linked_id\")\n",
    "y_train_or = train_or[[\"linked_id\"]]\n",
    "train.phone=train.phone.apply(str)\n",
    "test.phone=test.phone.apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Treat strings that are too short as missing values\n",
    "train.name=train.name.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "train.address=train.address.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "train.email=train.email.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "train.phone=train.phone.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "\n",
    "test.name=test.name.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "test.address=test.address.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "test.email=test.email.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "test.phone=test.phone.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "\n",
    "train=train.fillna('-1')\n",
    "test=test.fillna('-1')\n",
    "\n",
    "train = pd.concat([train,y_train_or['linked_id']],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###################\n",
    "#Dataset piccolino\n",
    "###################\n",
    "\n",
    "\n",
    "\n",
    "#m1=(train.linked_id=='15005501')|(train.linked_id=='13013105')|(train.linked_id=='13013089')|(train.linked_id=='13013092')|(train.linked_id=='13013071')|(train.linked_id=='13013058')|(train.linked_id=='13013055')|(train.linked_id=='13013041')|(train.linked_id=='13013044')\n",
    "#m2=(test.record_id=='15005501-TST-M')|(test.record_id=='13013105-T0-TST-CP')|(test.record_id=='13013089-TST-CP')|(test.record_id=='13013089-NV0-TST-CP')|(test.record_id=='13013089-T4-TST-CP')|(test.record_id=='13013092-NV0-TST-M')|(test.record_id=='13013092-TST-CP')|(test.record_id=='13013072-M1-TST-CP')|(test.record_id=='13013072-TST-CP')|(test.record_id=='10121215-M1')|(test.record_id=='10042968-T1') \n",
    "\n",
    "\n",
    "\n",
    "#train=train[m1]\n",
    "#test=test[m2]\n",
    "\n",
    "#train.reset_index(  drop=True, inplace=True)\n",
    "#test.reset_index( drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Create list from pd.Series\n",
    "test_name_id=test.record_id.tolist()\n",
    "test_ad_id=test.record_id.tolist()\n",
    "test_em_id=test.record_id.tolist()\n",
    "test_nu_id=test.record_id.tolist()\n",
    "\n",
    "\n",
    "train_name_id=train.record_id.tolist()\n",
    "train_ad_id=train.record_id.tolist()\n",
    "train_em_id=train.record_id.tolist()\n",
    "train_nu_id=train.record_id.tolist()\n",
    "\n",
    "\n",
    "test_name_list=test.name.tolist()\n",
    "test_address_list=test.address.tolist()\n",
    "test_email_list=test.email.tolist()\n",
    "test_number_list=test.phone.tolist()\n",
    "\n",
    "\n",
    "train_name_list=train.name.tolist()\n",
    "train_address_list=train.address.tolist()\n",
    "train_email_list=train.email.tolist()\n",
    "train_number_list=train.phone.tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Save indices of missing values to set them as zeros\n",
    "test_na_series=pd.Series(test_name_list)\n",
    "test_na_null=test_na_series[test_na_series=='-1'].index\n",
    "\n",
    "test_ad_series=pd.Series(test_address_list)\n",
    "test_ad_null=test_ad_series[test_ad_series=='-1'].index\n",
    "\n",
    "test_em_series=pd.Series(test_email_list)\n",
    "test_em_null=test_em_series[test_em_series=='-1'].index\n",
    "\n",
    "test_nu_series=pd.Series(test_number_list)\n",
    "test_nu_null=test_nu_series[test_nu_series=='-1'].index\n",
    "\n",
    "train_na_series=pd.Series(train_name_list)\n",
    "train_na_null=train_na_series[train_na_series=='-1'].index\n",
    "\n",
    "train_ad_series=pd.Series(train_address_list)\n",
    "train_ad_null=train_ad_series[train_ad_series=='-1'].index\n",
    "\n",
    "train_em_series=pd.Series(train_email_list)\n",
    "train_em_null=train_em_series[train_em_series=='-1'].index\n",
    "\n",
    "train_nu_series=pd.Series(train_number_list)\n",
    "train_nu_null=train_nu_series[train_nu_series=='-1'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create tfidf matrices\n",
    "analyzer='char_wb'\n",
    "n=3\n",
    "\n",
    "ct_vect = CountVectorizer(analyzer=analyzer, ngram_range=(n-1, n))\n",
    "vocab   = ct_vect.fit(test_name_list  + train_name_list).vocabulary_\n",
    "tfidf_vect  = TfidfVectorizer(vocabulary=vocab, analyzer=analyzer, ngram_range=(n-1, n))\n",
    "tf_idf_matrix_source_na=tfidf_vect.fit_transform(test_name_list)\n",
    "tf_idf_matrix_target_na=tfidf_vect.fit_transform(train_name_list)\n",
    "\n",
    "\n",
    "ct_vect_ad = CountVectorizer(analyzer=analyzer, ngram_range=(n-1, n))\n",
    "vocab_ad   = ct_vect_ad.fit(test_address_list  + train_address_list).vocabulary_\n",
    "tfidf_vect_ad  = TfidfVectorizer(vocabulary=vocab_ad, analyzer=analyzer, ngram_range=(n-1, n))\n",
    "tf_idf_matrix_source_ad=tfidf_vect_ad.fit_transform(test_address_list)\n",
    "tf_idf_matrix_target_ad=tfidf_vect_ad.fit_transform(train_address_list)\n",
    "\n",
    "ct_vect_em = CountVectorizer(analyzer=analyzer, ngram_range=(n-1, n))\n",
    "vocab_em   = ct_vect_em.fit(test_email_list  + train_email_list).vocabulary_\n",
    "tfidf_vect_em  = TfidfVectorizer(vocabulary=vocab_em, analyzer=analyzer, ngram_range=(n-1, n))\n",
    "tf_idf_matrix_source_em=tfidf_vect_em.fit_transform(test_email_list)\n",
    "tf_idf_matrix_target_em=tfidf_vect_em.fit_transform(train_email_list)\n",
    "\n",
    "\n",
    "ct_vect_nu = CountVectorizer(analyzer=analyzer, ngram_range=(n-1, n))\n",
    "vocab_nu   = ct_vect_nu.fit(test_number_list  + train_number_list).vocabulary_\n",
    "tfidf_vect_nu  = TfidfVectorizer(vocabulary=vocab_nu, analyzer=analyzer, ngram_range=(n-1, n))\n",
    "tf_idf_matrix_source_nu=tfidf_vect_nu.fit_transform(test_number_list)\n",
    "tf_idf_matrix_target_nu=tfidf_vect_nu.fit_transform(train_number_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set tfidf vectors of missing values in the test set as zeros cause their similarity scores are useless\n",
    "for a in test_na_null:\n",
    "    tf_idf_matrix_source_na.data[tf_idf_matrix_source_na.indptr[a]:tf_idf_matrix_source_na.indptr[a+1]]=0\n",
    "\n",
    "#for b in train_na_null:\n",
    "#    tf_idf_matrix_target_na.data[tf_idf_matrix_target_na.indptr[b]:tf_idf_matrix_target_na.indptr[b+1]]=0\n",
    "    \n",
    "    \n",
    "for c in test_ad_null:\n",
    "    tf_idf_matrix_source_ad.data[tf_idf_matrix_source_ad.indptr[c]:tf_idf_matrix_source_ad.indptr[c+1]]=0\n",
    "\n",
    "#for d in train_ad_null:\n",
    "#    tf_idf_matrix_target_ad.data[tf_idf_matrix_target_ad.indptr[d]:tf_idf_matrix_target_ad.indptr[d+1]]=0\n",
    "    \n",
    "    \n",
    "for e in test_em_null:\n",
    "    tf_idf_matrix_source_em.data[tf_idf_matrix_source_em.indptr[e]:tf_idf_matrix_source_em.indptr[e+1]]=0\n",
    "\n",
    "#for f in train_em_null:\n",
    "#    tf_idf_matrix_target_em.data[tf_idf_matrix_target_em.indptr[f]:tf_idf_matrix_target_em.indptr[f+1]]=0    \n",
    "\n",
    "    \n",
    "\n",
    "for g in test_nu_null:\n",
    "    tf_idf_matrix_source_nu.data[tf_idf_matrix_source_nu.indptr[g]:tf_idf_matrix_source_nu.indptr[g+1]]=0\n",
    "\n",
    "#for h in train_nu_null:\n",
    "#    tf_idf_matrix_target_nu.data[tf_idf_matrix_target_nu.indptr[h]:tf_idf_matrix_target_nu.indptr[h+1]]=0\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELFTIMED: 703.3167402744293\n"
     ]
    }
   ],
   "source": [
    "t1 = time.time()\n",
    "matches_em = awesome_cossim_top(tf_idf_matrix_source_em, tf_idf_matrix_target_em.transpose(),691440, 0)\n",
    "t = time.time()-t1\n",
    "print(\"SELFTIMED:\", t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELFTIMED: 1215.2100429534912\n"
     ]
    }
   ],
   "source": [
    "t1 = time.time()\n",
    "matches_ad = awesome_cossim_top(tf_idf_matrix_source_ad, tf_idf_matrix_target_ad.transpose(),691440, 0)\n",
    "t = time.time()-t1\n",
    "print(\"SELFTIMED:\", t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELFTIMED: 750.881795167923\n"
     ]
    }
   ],
   "source": [
    "t1 = time.time()\n",
    "matches_nu = awesome_cossim_top(tf_idf_matrix_source_nu, tf_idf_matrix_target_nu.transpose(),691440, 0)\n",
    "t = time.time()-t1\n",
    "print(\"SELFTIMED:\", t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELFTIMED: 2227.880414247513\n"
     ]
    }
   ],
   "source": [
    "t1 = time.time()\n",
    "matches_na = awesome_cossim_top(tf_idf_matrix_source_na, tf_idf_matrix_target_na.transpose(),691440, 0)\n",
    "t = time.time()-t1\n",
    "print(\"SELFTIMED:\", t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's create also csc format to set columns efficiently to zero \n",
    "name_csc=matches_na.tocsc()\n",
    "ad_csc=matches_ad.tocsc()\n",
    "em_csc=matches_em.tocsc()\n",
    "nu_csc=matches_nu.tocsc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now it's time to set similarity scores with nan train features as zeros\n",
    "for i in train_na_null:\n",
    "    name_csc.data[name_csc.indptr[i]:name_csc.indptr[i+1]]=0\n",
    "\n",
    "for j in train_ad_null:\n",
    "    ad_csc.data[ad_csc.indptr[j]:ad_csc.indptr[j+1]]=0\n",
    "\n",
    "for k in train_em_null:\n",
    "    em_csc.data[em_csc.indptr[k]:em_csc.indptr[k+1]]=0\n",
    "    \n",
    "for l in train_nu_null:\n",
    "    nu_csc.data[nu_csc.indptr[l]:nu_csc.indptr[l+1]]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's switch to coo format to sum the csc matrics\n",
    "name_coo=name_csc.tocoo()\n",
    "ad_coo=ad_csc.tocoo()\n",
    "em_coo=em_csc.tocoo()\n",
    "nu_coo=nu_csc.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's sum the matrices of similarity scores\n",
    "mat_fin=name_coo+ad_coo+em_coo+nu_coo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Creation of masks just for instances not excluded from top-n\n",
    "\n",
    "#na_w_mask = np.array(name_csc[name_csc.nonzero()] <=1)[0]\n",
    "#ad_w_mask = np.array(ad_csc[ad_csc.nonzero()] <=1)[0]\n",
    "#em_w_mask = np.array(em_csc[em_csc.nonzero()] <=1)[0]\n",
    "#nu_w_mask = np.array(nu_csc[nu_csc.nonzero()] <=1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#rows_na = name_csc.nonzero()[0][na_w_mask]\n",
    "#cols_na = name_csc.nonzero()[1][na_w_mask]\n",
    "\n",
    "#rows_ad = ad_csc.nonzero()[0][ad_w_mask]\n",
    "#cols_ad = ad_csc.nonzero()[1][ad_w_mask]\n",
    "\n",
    "#rows_em = em_csc.nonzero()[0][em_w_mask]\n",
    "#cols_em = em_csc.nonzero()[1][em_w_mask]\n",
    "\n",
    "#rows_nu = nu_csc.nonzero()[0][nu_w_mask]\n",
    "#cols_nu = nu_csc.nonzero()[1][nu_w_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weights_na=name_csc.copy()\n",
    "weights_ad=ad_csc.copy()\n",
    "weights_em=em_csc.copy()\n",
    "weights_nu=nu_csc.copy()\n",
    "\n",
    "#Uniform weights\n",
    "\n",
    "#weights_na[rows_na,cols_na]=1\n",
    "#weights_ad[rows_ad,cols_ad]=1\n",
    "#weights_em[rows_em,cols_em]=1\n",
    "#weights_nu[rows_nu,cols_nu]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Save indices of missing values to set them as zeros\n",
    "test_na_series=pd.Series(test_name_list)\n",
    "test_na_not_null=test_na_series[test_na_series!='-1'].index\n",
    "\n",
    "test_ad_series=pd.Series(test_address_list)\n",
    "test_ad_not_null=test_ad_series[test_ad_series!='-1'].index\n",
    "\n",
    "test_em_series=pd.Series(test_email_list)\n",
    "test_em_not_null=test_em_series[test_em_series!='-1'].index\n",
    "\n",
    "test_nu_series=pd.Series(test_number_list)\n",
    "test_nu_not_null=test_nu_series[test_nu_series!='-1'].index\n",
    "\n",
    "train_na_series=pd.Series(train_name_list)\n",
    "train_na_not_null=train_na_series[train_na_series!='-1'].index\n",
    "\n",
    "train_ad_series=pd.Series(train_address_list)\n",
    "train_ad_not_null=train_ad_series[train_ad_series!='-1'].index\n",
    "\n",
    "train_em_series=pd.Series(train_email_list)\n",
    "train_em_not_null=train_em_series[train_em_series!='-1'].index\n",
    "\n",
    "train_nu_series=pd.Series(train_number_list)\n",
    "train_nu_not_null=train_nu_series[train_nu_series!='-1'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/users/michele.bertoldi/anaconda3/lib/python3.6/site-packages/scipy/sparse/_index.py:126: SparseEfficiencyWarning: Changing the sparsity structure of a csc_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n"
     ]
    }
   ],
   "source": [
    "#Creation of correct weights matrices\n",
    "\n",
    "start=time.time()\n",
    "for j in train_ad_not_null:\n",
    "    weights_ad.data[weights_ad.indptr[j]:weights_ad.indptr[j+1]]=1\n",
    "print(time.time()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for j in train_em_not_null:\n",
    "    weights_em.data[weights_em.indptr[j]:weights_em.indptr[j+1]]=1\n",
    "    \n",
    "for j in train_na_not_null:\n",
    "    weights_na.data[weights_na.indptr[j]:weights_na.indptr[j+1]]=1   \n",
    "\n",
    "for j in train_nu_not_null:\n",
    "    weights_nu.data[weights_nu.indptr[j]:weights_nu.indptr[j+1]]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#---> COO format to execute sum of the weights matrices\n",
    "weights_na_coo=weights_na.tocoo()\n",
    "weights_ad_coo=weights_ad.tocoo()\n",
    "weights_em_coo=weights_em.tocoo()\n",
    "weights_nu_coo=weights_nu.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Matrix of weights\n",
    "weight_tot=weights_na_coo+weights_ad_coo+weights_em_coo+weights_nu_coo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's finally perform the division\n",
    "M3 = sparse_divide_nonzero(mat_fin, weight_tot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_linked_id_list=train.linked_id.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "match_df={}\n",
    "list_top=top_n_idx_sparse(M3,10)\n",
    "i=0\n",
    "for row_i in zip(test['record_id'],test['name']):\n",
    "    match_df[row_i[0]]=[train_linked_id_list[j] for j in list_top[i]]\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['12075168',\n",
       " '12096573',\n",
       " '12096573',\n",
       " '10080397',\n",
       " '10145923',\n",
       " '12104777',\n",
       " '12133698',\n",
       " '12133698',\n",
       " '10201125',\n",
       " '12133698']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "match_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_name = 'tfidf_independent_top10'\n",
    "csv_name += datetime.now().strftime('%b%d_%H-%M-%S')+'.csv'\n",
    "#pred_df.to_csv(\"../panama-oracle-f-2/\"+csv_name, index=False)\n",
    "\n",
    "pred_df_kaggle = prediction_dict_to_kaggle_df(match_df)\n",
    "\n",
    "pred_df_kaggle.to_csv(\"../panama-papers-polimi/\"+csv_name, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}