12.gen_feat_v4_bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_pickle('./pkl/train_data.pkl')\n",
    "test_data = pd.read_pickle('./pkl/valid_data.pkl')\n",
    "data = pd.concat([train_data, test_data]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_org_cross = pd.read_pickle('./feat/author_org_cross.pkl')\n",
    "keyword_venue_cross = pd.read_pickle('./feat/keyword_venue_cross.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "abstract_sims_st_bert = pd.read_pickle('./feat/abstract_sims_st_bert.pkl')\n",
    "keywords_sims_st_bert = pd.read_pickle('./feat/keywords_sims_st_bert.pkl')\n",
    "title_sims_st_bert = pd.read_pickle('./feat/title_sims_st_bert.pkl')\n",
    "venue_sims_st_bert = pd.read_pickle('./feat/venue_sims_st_bert.pkl')\n",
    "authors_sims_st_bert = pd.read_pickle('./feat/authors_sims_st_bert.pkl')\n",
    "orgs_sims_st_bert = pd.read_pickle('./feat/orgs_sims_st_bert.pkl')\n",
    "feat_v3_bert = pd.read_pickle('./feat/feat_v3_bert.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_feat = [abstract_sims_st_bert, keywords_sims_st_bert, title_sims_st_bert, venue_sims_st_bert, authors_sims_st_bert, orgs_sims_st_bert, feat_v3_bert]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['author_id', 'author_name', 'author_org', 'label', 'paper_id',\n",
      "       'author_org_in_orgs_b_times', 'author_interset_num',\n",
      "       'author_interset_num/paper_ids_len', 'venue_a_in_venue_b_num',\n",
      "       'keywords_interset_num', 'venue_a_in_venue_b_num/paper_ids_len',\n",
      "       'keywords_interset_num/paper_ids_len', 'abstract_sims_min',\n",
      "       'abstract_sims_max', 'abstract_sims_mean', 'abstract_sims_std',\n",
      "       'abstract_sims_mm2', 'keywords_sims_min', 'keywords_sims_max',\n",
      "       'keywords_sims_mean', 'keywords_sims_std', 'keywords_sims_mm2',\n",
      "       'title_sims_min', 'title_sims_max', 'title_sims_mean', 'title_sims_std',\n",
      "       'title_sims_mm2', 'venue_sims_min', 'venue_sims_max', 'venue_sims_mean',\n",
      "       'venue_sims_std', 'venue_sims_mm2', 'authors_sims_min',\n",
      "       'authors_sims_max', 'authors_sims_mean', 'authors_sims_std',\n",
      "       'authors_sims_mm2', 'orgs_sims_min', 'orgs_sims_max', 'orgs_sims_mean',\n",
      "       'orgs_sims_std', 'orgs_sims_mm2', 'paper_num', 'orgs_num',\n",
      "       'orgs_sims_max2', 'orgs_sims_min2', 'orgs_sims_mean2',\n",
      "       'orgs_sims_std2'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([data, author_org_cross, keyword_venue_cross] + bert_feat, axis=1)\n",
    "print(data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_to_rank = ['paper_num', 'author_org_in_orgs_b_times', 'author_interset_num',\n",
    "       'author_interset_num/paper_ids_len', 'venue_a_in_venue_b_num',\n",
    "       'keywords_interset_num', 'venue_a_in_venue_b_num/paper_ids_len',\n",
    "       'keywords_interset_num/paper_ids_len', \n",
    "       'abstract_sims_min',\n",
    "       'abstract_sims_max', 'abstract_sims_mean', 'abstract_sims_std',\n",
    "       'abstract_sims_mm2', 'keywords_sims_min', 'keywords_sims_max',\n",
    "       'keywords_sims_mean', 'keywords_sims_std', 'keywords_sims_mm2',\n",
    "       'title_sims_min', 'title_sims_max', 'title_sims_mean', 'title_sims_std',\n",
    "       'title_sims_mm2', 'venue_sims_min', 'venue_sims_max', 'venue_sims_mean',\n",
    "       'venue_sims_std', 'venue_sims_mm2', 'authors_sims_min',\n",
    "       'authors_sims_max', 'authors_sims_mean', 'authors_sims_std',\n",
    "       'authors_sims_mm2', 'orgs_sims_min', 'orgs_sims_max', 'orgs_sims_mean',\n",
    "       'orgs_sims_std', 'orgs_sims_mm2', \n",
    "       'orgs_sims_max2', 'orgs_sims_min2', 'orgs_sims_mean2',\n",
    "       'orgs_sims_std2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['pair_name'] = data['author_name'] + '_' + data['paper_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "data['pair_name'] = LabelEncoder().fit_transform(data['pair_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 paper_num\n",
      "1 author_org_in_orgs_b_times\n",
      "2 author_interset_num\n",
      "3 author_interset_num/paper_ids_len\n",
      "4 venue_a_in_venue_b_num\n",
      "5 keywords_interset_num\n",
      "6 venue_a_in_venue_b_num/paper_ids_len\n",
      "7 keywords_interset_num/paper_ids_len\n",
      "8 abstract_sims_min\n",
      "9 abstract_sims_max\n",
      "10 abstract_sims_mean\n",
      "11 abstract_sims_std\n",
      "12 abstract_sims_mm2\n",
      "13 keywords_sims_min\n",
      "14 keywords_sims_max\n",
      "15 keywords_sims_mean\n",
      "16 keywords_sims_std\n",
      "17 keywords_sims_mm2\n",
      "18 title_sims_min\n",
      "19 title_sims_max\n",
      "20 title_sims_mean\n",
      "21 title_sims_std\n",
      "22 title_sims_mm2\n",
      "23 venue_sims_min\n",
      "24 venue_sims_max\n",
      "25 venue_sims_mean\n",
      "26 venue_sims_std\n",
      "27 venue_sims_mm2\n",
      "28 authors_sims_min\n",
      "29 authors_sims_max\n",
      "30 authors_sims_mean\n",
      "31 authors_sims_std\n",
      "32 authors_sims_mm2\n",
      "33 orgs_sims_min\n",
      "34 orgs_sims_max\n",
      "35 orgs_sims_mean\n",
      "36 orgs_sims_std\n",
      "37 orgs_sims_mm2\n",
      "38 orgs_sims_max2\n",
      "39 orgs_sims_min2\n",
      "40 orgs_sims_mean2\n",
      "41 orgs_sims_std2\n"
     ]
    }
   ],
   "source": [
    "ranks = []\n",
    "for i, c in enumerate(col_to_rank):\n",
    "    print(i, c)\n",
    "    rank_a = data.groupby('pair_name')[c].rank(ascending=True).rename('%s_rank_a' % c)\n",
    "    rank_b = data.groupby('pair_name')[c].rank(ascending=False).rename('%s_rank_b' % c)\n",
    "    ranks.append(rank_a)\n",
    "    ranks.append(rank_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = pd.concat([data] + ranks, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [c + '_rank_a' for c in col_to_rank] + [c + '_rank_b' for c in col_to_rank]\n",
    "data[cols].to_pickle('./feat/rank_feat_bert.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}