check_one_person_1_feat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "from multiprocessing import Pool\n",
    "from functools import partial\n",
    "import math\n",
    "from operator import itemgetter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def timeit(method):\n",
    "    \"\"\"\n",
    "    Standard Python decorator that measures the execution time of a method;\n",
    "    \"\"\"\n",
    "    def timed(*args, **kw):\n",
    "        start = time.time()\n",
    "        result = method(*args, **kw)\n",
    "        end = time.time()\n",
    "        \n",
    "        print(f\"{method.__name__}:  {(end - start):.2f} s\")\n",
    "        return result\n",
    "    return timed\n",
    "\n",
    "#############################\n",
    "#############################\n",
    "\n",
    "@timeit\n",
    "def load_training_data(data_path: str, row_subset: float=1, train_split: float=0.7, shuffle: bool=False, seed=None):\n",
    "    '''\n",
    "    Load the training set and divide it into training and test splits.\n",
    "    \"LinkedID\" is the value that we want to predict\n",
    "    :param data_path: path to the dataset to load;\n",
    "    :param row_subset: use only the specified fraction of rows in the dataset (value in (0, 1]);\n",
    "    :param train_split: fraction of rows placed in the training set;\n",
    "    :param shuffle: if True, shuffle the rows before splitting or subsetting the data;\n",
    "    '''\n",
    "    if row_subset <= 0 or row_subset > 1:\n",
    "        row_subset = 1\n",
    "    \n",
    "    data = read_file(training_file, set_record_id_as_index=True)\n",
    "    if shuffle:\n",
    "        data = data.sample(frac=1, random_state=seed)\n",
    "    # Obtain the specified subset of rows;\n",
    "    data = data.iloc[:int(np.ceil(len(data) * row_subset))]\n",
    "        \n",
    "    X = data.drop(columns=\"linked_id\")\n",
    "    y = data[\"linked_id\"]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=train_split, shuffle=shuffle, random_state =seed)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "#############################\n",
    "#############################\n",
    "\n",
    "@timeit\n",
    "def clean_data(data: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Perform cleaning of the dataset, e.g. lowercase strings, fill missing values, etc...\n",
    "    \"\"\"\n",
    "    cleaned_data = data.copy()\n",
    "    for c in [\"name\", \"type\", \"address\", \"phone\", \"email\", \"modification\"]:\n",
    "        cleaned_data[c] = cleaned_data[c].str.lower()\n",
    "    \n",
    "    return cleaned_data\n",
    "\n",
    "#############################\n",
    "#############################\n",
    "    \n",
    "@timeit\n",
    "def index_data(data: list) -> list:\n",
    "    \"\"\"\n",
    "    Manipulate the data to create indices that speed up the computation. \n",
    "    \n",
    "    The dataframes to be indexed are passed with a list of pd.DataFrames.\n",
    "    Be careful not to leak information from the train dataset to the test dataset!\n",
    "    \n",
    "    In this case, replace all strings with unique integer values,\n",
    "    so that comparing rows for equality is much faster;\n",
    "    \"\"\"\n",
    "    \n",
    "    # Obtain the original ending of each dataframe.\n",
    "    # Prepend a 0 to indicate the start of the first dataframe;\n",
    "    lengths = [0] + [len(x) for x in data]\n",
    "    lengths = np.cumsum(lengths)\n",
    "    # Join together all the input data;\n",
    "    concatenated_data = pd.concat(data)\n",
    "        \n",
    "    for c in [\"email_domain\",\"type\", \"address_country\",\"phone_prefix\",\"email_suffix\",\"modification\"]:\n",
    "        concatenated_data[c] = pd.factorize(concatenated_data[c])[0]\n",
    "    \n",
    "    # Split the input data;\n",
    "    indexed_data = []\n",
    "    for i in range(len(lengths) - 1):\n",
    "        indexed_data += [concatenated_data.iloc[lengths[i]:lengths[i + 1], :]]\n",
    "    \n",
    "    return indexed_data\n",
    "\n",
    "#############################\n",
    "#############################\n",
    "\n",
    "\n",
    "    \n",
    "def predict_record_fast(train: tuple, record: np.array, K: int=10) -> list:\n",
    "    \"\"\"\n",
    "    Given a record and a training set, find the records in the training set that best match\n",
    "    the input record. Predictions can be done using very simple unsupervised algorithms,\n",
    "    as done in this example, or computed with some fancy ML model;\n",
    "    \"\"\"\n",
    "    # In this simple model, look for the number of identical columns for each training record.\n",
    "    # Skip the last column (type of modification);\n",
    "    X_train,y_train=train[0],train[1]\n",
    "    #identical_values_per_row = pd.Series(np.array(record[3:] == X_train[:,3:]).sum(axis=1))\n",
    "\n",
    "    # Fuzzy match\n",
    "    dataframecolumn0 = pd.DataFrame([record])\n",
    "    dataframecolumn0.columns = ['Match0']\n",
    "    \n",
    "    compare0 = pd.DataFrame(X_train[:,0])\n",
    "    compare0.columns = ['compare0']\n",
    "    \n",
    "    dataframecolumn0['Key0'] = 1\n",
    "    compare0['Key0'] = 1\n",
    "    combined_dataframe0 = dataframecolumn0.merge(compare0,on=\"Key0\",how=\"left\")\n",
    "    \n",
    "    identical_values_per_row = pd.Series(partial_match_vector_complessivo(combined_dataframe0['Match0'],combined_dataframe0['compare0']))\n",
    "    '''\n",
    "    # Fuzzy address match\n",
    "    dataframecolumn1 = pd.DataFrame([record[1]])\n",
    "    dataframecolumn1.columns = ['Match1']\n",
    "    \n",
    "    compare1 = pd.DataFrame(X_train[:,1])\n",
    "    compare1.columns = ['compare1']\n",
    "    \n",
    "    dataframecolumn1['Key1'] = 1\n",
    "    compare1['Key1'] = 1\n",
    "    combined_dataframe1 = dataframecolumn1.merge(compare1,on=\"Key1\",how=\"left\")\n",
    "    \n",
    "    identical_values_per_row += partial_match_vector_address(combined_dataframe1['Match1'],combined_dataframe1['compare1'])\n",
    "\n",
    "    # Fuzzy email_first_part match\n",
    "    dataframecolumn2 = pd.DataFrame([record[3]])\n",
    "    dataframecolumn2.columns = ['Match2']\n",
    "    \n",
    "    compare2 = pd.DataFrame(X_train[:,3])\n",
    "    compare2.columns = ['compare2']\n",
    "    \n",
    "    dataframecolumn2['Key2'] = 1\n",
    "    compare2['Key2'] = 1\n",
    "    combined_dataframe2 = dataframecolumn2.merge(compare2,on=\"Key2\",how=\"left\")\n",
    "    \n",
    "    identical_values_per_row += partial_match_vector_email_first(combined_dataframe2['Match2'],combined_dataframe2['compare2'])\n",
    "\n",
    "    #Fuzzy email_domain match\n",
    "    dataframecolumn3 = pd.DataFrame([record[2]])\n",
    "    dataframecolumn3.columns = ['Match3']\n",
    "    \n",
    "    compare3 = pd.DataFrame(X_train[:,2])\n",
    "    compare3.columns = ['compare3']\n",
    "    \n",
    "    dataframecolumn3['Key3'] = 1\n",
    "    compare3['Key3'] = 1\n",
    "    combined_dataframe3 = dataframecolumn3.merge(compare3,on=\"Key3\",how=\"left\")\n",
    "    \n",
    "    identical_values_per_row += partial_match_vector_number_no_prefix(combined_dataframe3['Match3'],combined_dataframe3['compare3'])\n",
    "    '''\n",
    "    \n",
    "    # Obtain the K rows with the most matches;\n",
    "    best_matches = identical_values_per_row.sort_values(ascending=False)[:K]    \n",
    "    # Retrieve the original record IDs from the training set;\n",
    "    #print(best_matches)\n",
    "    return list(zip(list(y_train.loc[best_matches.index.values]) ,list(best_matches)))\n",
    "\n",
    "\n",
    "def predict_er_fast(X_train, y_train,info,soglia, K=10) -> dict:\n",
    "#def predict_er_fast(train, X_test, K=10) -> dict:\n",
    "    \"\"\"\n",
    "    Given a training dataset and a test dataset, obtain the top-K predictions \n",
    "    for each record in the test dataset;\n",
    "    \"\"\"\n",
    "\n",
    "    # Store for each record the list of predictions;\n",
    "    predictions = {}\n",
    "    start = time.time()\n",
    "    \n",
    "    # Extract the matrix that represent the data.\n",
    "    # Skip the last column;\n",
    "    #X_train_matrix = X_train.values[:, :-1]\n",
    "    \n",
    "    # Also reindex the y column, as we lose the original index when doing a comparison using matrices;\n",
    "    y_train_vector = y_train.reset_index(drop=True)\n",
    "    #X_train=X_train.reset_index(drop=True)\n",
    "    n_threads=18\n",
    "    n_reduced=math.ceil(len(X_train)/float(n_threads))\n",
    "    chunks=[X_train.iloc[i*n_reduced:(i+1)*n_reduced,1:].values for i in range(n_threads)]\n",
    "    chunks_y=[y_train_vector.iloc[i*n_reduced:(i+1)*n_reduced].reset_index(drop=True) for i in range(n_threads)]\n",
    "    print(chunks[0])\n",
    "    # Compute a prediction for each record;\n",
    "    i=0\n",
    "    for i in range(1): #,X_test['email_domain'],X_test['type'],X_test['address_country'],X_test['phone_prefix'],X_test['email_suffix']):\n",
    "        # Extract values from the current row;\n",
    "        #row_i=(name,address,phone,mail)\n",
    "        #row_i_vector = np.array(row_i)\n",
    "        #print(row_i_vector)\n",
    "        # Find the best matching record for the current record;\n",
    "        pool=Pool(n_threads)\n",
    "        input_data = tuple(zip(chunks,chunks_y))\n",
    "        appoggio =[x[:] for x in pool.map(partial(predict_record_fast, record=info,K=10),input_data)]\n",
    "        appoggio_2=[item for sublist  in appoggio for item in sublist]\n",
    "        #print(appoggio_2)\n",
    "        appoggio_3=sorted(appoggio_2,key=lambda x : float(x[1]),reverse=True)\n",
    "        predictions[0]=[i for i in appoggio_3 if i[1]>=soglia][:K]\n",
    "    return predictions\n",
    "\n",
    "\n",
    "@timeit\n",
    "def prediction_dict_to_df(predictions: dict) -> pd.DataFrame:\n",
    "    # Turn the prediction dict into a series of tuples;\n",
    "    results = []\n",
    "    for query_id, pred_list in predictions.items():\n",
    "        for p in pred_list:\n",
    "            results += [[query_id, p]]\n",
    "    return pd.DataFrame(results, columns=[\"queried_record_id\", \"predicted_record_id\"])\n",
    "\n",
    "@timeit\n",
    "def prediction_dict_to_kaggle_df(predictions: dict) -> pd.DataFrame:\n",
    "    # Turn the prediction dict into a series of tuples;\n",
    "    results = []\n",
    "    for query_id, pred_list in predictions.items():\n",
    "        results += [[query_id, \" \".join(pred_list)]]\n",
    "    return pd.DataFrame(results, columns=[\"queried_record_id\", \"predicted_record_id\"])\n",
    "\n",
    "@timeit\n",
    "def kaggle_sol_to_df(kaggle_df: pd.DataFrame) -> pd.DataFrame:\n",
    "    kaggle_df_indexed = kaggle_df.set_index(\"queried_record_id\")\n",
    "    results = []\n",
    "    for query_id, pred_list in kaggle_df_indexed.iterrows():\n",
    "        results += [[query_id, pred] for pred in pred_list[\"predicted_record_id\"].split(\" \")]\n",
    "    return pd.DataFrame(results, columns=[\"queried_record_id\", \"predicted_record_id\"])\n",
    "\n",
    "def read_file(path: str, set_record_id_as_index: bool=False) -> pd.DataFrame:\n",
    "    return pd.read_csv(path, dtype=str, escapechar=\"\\\\\", index_col=\"record_id\" if set_record_id_as_index else None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_filez = \"../panama-papers-polimi/data/entity-resolution_advanced-topics-training_data.csv\"\n",
    "train = read_file(training_filez, set_record_id_as_index=False)\n",
    "#X_train_parra = train.drop(columns=\"linked_id\")\n",
    "y_train = train[\"record_id\"]\n",
    "#X_train, X_test, y_train, y_test = load_training_data(training_file, shuffle=True, row_subset=0.01, seed=42)\n",
    "\n",
    "trainn=pd.read_csv('../panama-papers-polimi/data/panama_train_expanded_2.csv')\n",
    "#trainn = read_file(training_file, set_record_id_as_index=True)\n",
    "\n",
    "testt=pd.read_csv('../panama-papers-polimi/data/panama_train_expanded_2.csv')\n",
    "#testt = read_file(testing_file, set_record_id_as_index=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=trainn.copy()\n",
    "X_test=testt.copy()\n",
    "\n",
    "X_train=X_train[[\"record_id\",\"name\",\"address\",\"email\",\"phone\"]] #\"email_domain\",\"type\", \"address_country\",\"phone_prefix\",\"email_suffix\",\"modification\"]]\n",
    "X_test=X_test[[\"record_id\",\"name\",\"address\",\"email\",\"phone\"]] #\"email_domain\",\"type\" ,\"address_country\",\"phone_prefix\",\"email_suffix\",\"modification\"]]\n",
    "\n",
    "\n",
    "X_train.phone=X_train.phone.apply(str)\n",
    "X_test.phone=X_test.phone.apply(str)\n",
    "\n",
    "X_train=X_train.fillna('-1')\n",
    "X_test=X_test.fillna('-1')\n",
    "\n",
    "#Treat strings that are too short as missing values\n",
    "X_train.name=X_train.name.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_train.address=X_train.address.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_train.email=X_train.email.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_train.phone=X_train.phone.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "\n",
    "X_test.name=X_test.name.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_test.address=X_test.address.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_test.email=X_test.email.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "X_test.phone=X_test.phone.apply(lambda x : '-1' if len(x)<=2 else x)\n",
    "\n",
    "X_train=X_train.fillna('-1')\n",
    "X_test=X_test.fillna('-1')\n",
    "\n",
    "\n",
    "\n",
    "#Create new merged features from name,address,phone,email\n",
    "\n",
    "X_train_3=X_train.copy()\n",
    "X_train_3.set_index('record_id',inplace=True)\n",
    "X_train.name=X_train.name.apply(lambda x : '' if x=='-1' else x)\n",
    "X_train.address=X_train.address.apply(lambda x : '' if x=='-1' else x)\n",
    "X_train.email=X_train.email.apply(lambda x : '' if x=='-1' else x)\n",
    "X_train.phone=X_train.phone.apply(lambda x : '' if x=='-1' else x)\n",
    "\n",
    "X_test.name=X_test.name.apply(lambda x : '' if x=='-1' else x)\n",
    "X_test.address=X_test.address.apply(lambda x : '' if x=='-1' else x)\n",
    "X_test.email=X_test.email.apply(lambda x : '' if x=='-1' else x)\n",
    "X_test.phone=X_test.phone.apply(lambda x : '' if x=='-1' else x)\n",
    "\n",
    "X_train=X_train[['record_id','name','address','email','phone']]\n",
    "X_train['complessivo']=X_train['name']+' '+X_train['address']+' '+X_train['email']+' '+X_train['phone']\n",
    "#X_train.set_index('record_id',inplace=True)\n",
    "\n",
    "X_test['complessivo']=X_test['name']+' '+X_test['address']+' '+X_test['email']+' '+X_test['phone']\n",
    "\n",
    "\n",
    "X_train=X_train[['record_id','complessivo']]\n",
    "X_test=X_test[['record_id','complessivo']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def partial_match_name(x,y):\n",
    "    return(fuzz.ratio(x,y)*0.092)\n",
    "partial_match_vector_name = np.vectorize(partial_match_name)\n",
    "\n",
    "def partial_match_address(x,y):\n",
    "    return(fuzz.ratio(x,y)*0.002)\n",
    "partial_match_vector_address = np.vectorize(partial_match_address)\n",
    "\n",
    "def partial_match_email_first(x,y):\n",
    "    return(fuzz.ratio(x,y)*0.002)\n",
    "partial_match_vector_email_first = np.vectorize(partial_match_email_first)\n",
    "\n",
    "def partial_match_number_no_prefix(x,y):\n",
    "    return(fuzz.ratio(x,y)*0.004)\n",
    "partial_match_vector_number_no_prefix = np.vectorize(partial_match_number_no_prefix)\n",
    "\n",
    "def partial_match_complessivo(x,y):\n",
    "    return(fuzz.token_set_ratio(x,y)*0.1)\n",
    "partial_match_vector_complessivo = np.vectorize(partial_match_complessivo)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome,please type the personal information of the person you need to check\n",
      "NAME :luca nardini\n",
      "ADDRESS :\n",
      "PHONE :\n",
      "MAIL :\n"
     ]
    }
   ],
   "source": [
    "print('Welcome,please type the personal information of the person you need to check')\n",
    "\n",
    "name = input('NAME :')\n",
    "address = input('ADDRESS :')\n",
    "phone= input('PHONE :')\n",
    "mail=input('MAIL :')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please insert a threshold value between 0 and 10 :0\n",
      "Lastly, please insert the number of matching records you would like to retrieve :20\n"
     ]
    }
   ],
   "source": [
    "soglia = input('Please insert a threshold value between 0 and 10 :')\n",
    "K=input('Lastly, please insert the number of matching records you would like to retrieve :')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['ardia enterprisesmltd  imquires@zoho.vo 4102971003602']\n",
      " ['andre almeida blanco la rua pelotas 209 apartamento 72 bloco a en el municipio de sao paulo estado de sao paulo cep 0k012-000  ']\n",
      " ['moonta holdings ltd   65013464145']\n",
      " ...\n",
      " ['sterling mckay partners assets sa  sterlingmckaypartnersassetssa@outlook.de ']\n",
      " ['tinley holdings limited   446818347899']\n",
      " ['terrace property holdings ltd  inquires@icloud.de ']]\n"
     ]
    }
   ],
   "source": [
    "info=name+' '+address+' '+phone+' '+mail\n",
    "info=np.array(info)\n",
    "predictions = predict_er_fast(X_train, y_train,info,float(soglia),int(K))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the list of the best matching candidates and their similarity scores you asked for : \n",
      "[('12223689-M0', 10.0), ('12223689-NV0', 10.0), ('12223689', 10.0), ('12223689-M1', 10.0), ('12223689-M2', 10.0), ('11002948-M2', 7.0), ('13009323-M2', 6.7), ('12112309-M0', 6.7), ('13009323-M1', 6.7), ('13009323-M0', 6.7), ('12150053-M1', 6.7), ('12112309-M1', 6.7), ('10158628-M0', 6.4), ('10158628', 6.4), ('10158628-T0', 6.4), ('10158628-T1', 6.4), ('12043922-T0', 6.2), ('10052529', 6.2), ('12223841-M0', 6.2), ('12089129', 6.2)]\n"
     ]
    }
   ],
   "source": [
    "print('This is the list of the best matching candidates and their similarity scores you asked for : ')\n",
    "print(predictions[0])\n",
    "array=[]\n",
    "for j in predictions[0]:\n",
    "    if j[0]!='':\n",
    "        array.append(j[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "And now you can check the personal information of the best matches : \n",
      "record_id\n",
      "12223689-M0          luca nardini\n",
      "12223689-NV0         luca nardini\n",
      "12223689             luca nardini\n",
      "12223689-M1          luca nardini\n",
      "12223689-M2          luca nardini\n",
      "11002948-M2           srini lanka\n",
      "13009323-M2                lu nan\n",
      "12112309-M0       luca gallinelli\n",
      "13009323-M1                lu nan\n",
      "13009323-M0                lu nan\n",
      "12150053-M1          luca sandoli\n",
      "12112309-M1       luca gallinelli\n",
      "10158628-M0      luchy trading sa\n",
      "10158628         luchy trading sa\n",
      "10158628-T0      luchy trading sa\n",
      "10158628-T1      luchy trading sa\n",
      "12043922-T0        lua mariadmena\n",
      "10052529           sardinali corp\n",
      "12223841-M0        laurent nordin\n",
      "12089129        lucila piacentini\n",
      "Name: name, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print('And now you can check the personal information of the best matches : ')\n",
    "print(X_train_3.loc[array,'name'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}