build_a_profile-no-nsf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ideas for statistics\n",
    "\n",
    "\n",
    "#### 3 types of authorship model scores (full, 1/n, 1/position)\n",
    "* Mean Scores across years (x)\n",
    "* Max Scores across year & Year of Max ( )\n",
    "* Average value in last 7 years  ( )\n",
    "* Most productive co-author (x)\n",
    "\n",
    "#### Remaining Analysis from 1/position\n",
    "* \"RI\" Conf sub-score (X)\n",
    "* \"Top Graphics\",\"Top Vision\", \"Top Robotics\", \"Top ML\", \"Other\" sub-scores (x)\n",
    "* Average number of authors (x)\n",
    "* Average Author Position (x)\n",
    "* Average & Median \"quality\" of collab ( )\n",
    "* Current Affiliation (x)\n",
    "* Total number of collab ( )\n",
    "* Top 3 collabs (x)\n",
    "* Top 3 conferences from generated value ( )\n",
    "* Career length (x)\n",
    "* Number of collabs w/ more than 4 papers\n",
    "\n",
    "#### Advanced Stats from 1/n\n",
    "* 5 unlabeled variants of plus-minus (w/ intercept) (x)\n",
    "* 5 unlabeled variants of plius-minus (w/o intercept) (x)\n",
    "\n",
    "#### NSF Data\n",
    "* Total number of grants (x)\n",
    "* Total grant money (x)\n",
    "* fractional grant money (x)\n",
    "* grant money of collabs ( )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import fnmatch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import gzip\n",
    "import pickle\n",
    "import csv\n",
    "import scipy.sparse\n",
    "Xauth = None\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the update to work despite the broken scipy documentation\n",
    "try:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a.update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix.update\n",
    "except:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a._update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix._update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('useful_venue_list.pkl.gz','rb') as fp:\n",
    "    all_venues = pickle.load(fp)\n",
    "with gzip.open('useful_authors_list.pkl.gz','rb') as fp:\n",
    "    all_authors = pickle.load(fp)\n",
    "with gzip.open('useful_papers.pkl.gz','rb') as fp:\n",
    "    all_papers = pickle.load(fp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "min_year = all_papers[0][6]\n",
    "max_year = all_papers[-1][6]\n",
    "span_years = max_year - min_year + 1\n",
    "print(min_year,max_year,span_years)\n",
    "conf_idx = {v:i for i,v in enumerate(all_venues)}\n",
    "name_idx = {v:i for i,v in enumerate(all_authors)}\n",
    "n_confs = len(all_venues)\n",
    "n_auths = len(all_authors)\n",
    "n_papers = len(all_papers)\n",
    "print(n_confs,n_auths,n_papers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoreV = {}\n",
    "clf =  np.load('clf_gold.pkl.npy')\n",
    "years_per_conf = clf.shape[0]//n_confs\n",
    "YEAR_BLOCKS = span_years//years_per_conf\n",
    "clf[2323]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for FI in [False,True]:\n",
    "    scoreV['_apm' + str(FI)] = np.load('apm'+str(FI) + '.npy')\n",
    "    scoreV['pw_apm' + str(FI)] = np.load('pwapm'+str(FI) + '.npy')\n",
    "    scoreV['pweff_apm' + str(FI)] = np.load('pweffapm'+str(FI) + '.npy')\n",
    "    scoreV['pwunk_apm' + str(FI)] = np.load('pwunkapm'+str(FI) + '.npy')\n",
    "    print(scoreV['pwunk_apm' + str(FI)].shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if True:\n",
    "    try:\n",
    "        import gzip\n",
    "        import pickle\n",
    "        with gzip.open('scoresV2.pkl.gz','rb') as fp:\n",
    "            scoreV = pickle.load(fp)\n",
    "    except:\n",
    "        print('failed!')\n",
    "    PROCESS_DATA = len(scoreV) < 13\n",
    "    print(scoreV['pwunk_apm' + str(FI)].shape)\n",
    "    USE_LIMITS = False\n",
    "else:\n",
    "    PROCESS_DATA = True\n",
    "    USE_LIMITS = False\n",
    "    last_years = np.load('last_years.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(k,_.shape) for k,_ in scoreV.items() if _.shape[0] == 2337504]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('top_ri_Metric.json','rt') as fp:\n",
    "    interesting_set = set(json.load(fp))\n",
    "print(len(interesting_set))\n",
    "curious_names = ['Xiaolong Wang 0004','Judy Hoffman','Paris Siminelakis','Roie Levin','Leonid Keselman',\n",
    "                 'Nicholas Rhinehart','Vincent Sitzmann','Siddharth Ancha','Xingyu Lin',\n",
    "                 'Humphrey Hu','Aditya Dhawale','Nick Gisolfi','Andrey Kurenkov','Micah Corah',\n",
    "                 'David F. Fouhey','Chelsea Finn','Akshara Rai','Ankit Bhatia','Xuning Yang',\n",
    "                 'Lerrel Pinto','Graeme Best','Alexander Spitzer','Roberto Shu','Amir Abboud',\n",
    "                 'Justin Johnson','Kumar Shaurya Shankar','Ellen A. Cappo',\n",
    "                 'Amir Roshan Zamir','Dominik Peters','Jonathan T. Barron','Dorsa Sadigh','Derek Hoiem','Vaggos Chatziafratis',\n",
    "                 'Brian Okorn','David Held']\n",
    "#interesting_set = set(curious_names)\n",
    "with open('ri_cand_names.pkl','rb') as fp:\n",
    "    interesting_set = set(    pickle.load(fp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#interesting_set.remove('David P. Hayden')\n",
    "#interesting_set.add('David S. Hayden')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('ri_cand_names.pkl','wb') as fp:\n",
    "#     pickle.dump(interesting_set,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "words_count = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    for w in title.lower().replace(',','').replace('-',' ').replace(':','').replace('.','').split():\n",
    "        words_count[w] = 1+words_count.get(w,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted([(v,k) for k,v in words_count.items()],reverse=True)[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_total =sum(words_count.values())\n",
    "word_person = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    wc = []\n",
    "    for w in title.lower().replace(',','').replace('-',' ').replace(':','').replace('.','').split():\n",
    "        wc.append(words_count.get(w,0))\n",
    "    wc = [_ for _ in wc if _ < words_count['an'] and _ > 1]\n",
    "    if len(wc) > 0:\n",
    "        wc = np.mean(wc)\n",
    "        for a in authors:\n",
    "            word_person[a] = word_person.get(a,[]) + [wc]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_person = {k: np.mean(np.log(v)) for k,v in word_person.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_cand = [\n",
    " 'Changliu Liu',\n",
    " \"Matthew O'Toole\",\n",
    " \"Jun-Yan Zhu\",\n",
    "  \"Wenzhen Yuan\",\n",
    "    \"Oliver Kroemer\",\n",
    "    \"James McCann\",\n",
    "    \"Ioannis Gkioulekas\",\n",
    "    \"Keenan Crane\",\n",
    "    \"Henny Admoni\",\n",
    "    \"Shubham Tulsiani\",\n",
    "    \"Melisa Orta Martinez\",\n",
    " 'Fatma Zeynep Temel',\n",
    "    \"Deepak Pathak\",\n",
    "    \"David Held\",\n",
    "    \"Zachary Manchester\",\n",
    "]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmu_uni = pd.read_csv('other_ranks/cmu_faculty.csv')\n",
    "cmu_uni = cmu_uni.fillna('Other')\n",
    "cmu_uni = cmu_uni[cmu_uni.dept == 'RI']\n",
    "uni_names = set(list(cmu_uni.name))\n",
    "for n in prev_cand:\n",
    "    uni_names.add(n)\n",
    "print(len(uni_names))\n",
    "conf_counts = {}\n",
    "conf_counts_value = {}\n",
    "\n",
    "#interesting_set = uni_names\n",
    "\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    if year < 2004:\n",
    "        continue\n",
    "    n = len(authors)\n",
    "    for a in authors:\n",
    "        if a in uni_names:\n",
    "            conf_counts[venue] = 1/n + conf_counts.get(venue,0)\n",
    "            conf_counts_value[venue] = clf[years_per_conf*(conf_idx[venue]) + (year-min_year)//YEAR_BLOCKS]/n + conf_counts_value.get(venue,0)\n",
    "conf_counts_value = {k: v/conf_counts[k] for k,v in conf_counts_value.items()}\n",
    "ri_fav_confs = [(conf_counts[_[1]]*conf_counts_value[_[1]],_[1],conf_counts[_[1]],conf_counts_value[_[1]]) for _ in sorted([(v,k) for k,v in conf_counts.items() if v > 0],reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ri_confs = [_[1] for _ in sorted(ri_fav_confs,reverse=True) if _[-2] >= 1.25]\n",
    "#confs_to_filter =['ICRA','IROS','Robotics: Science and Systems']\n",
    "ri_confs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if PROCESS_DATA:\n",
    "    auth_years = np.ones((n_auths,2)) * np.array([3000,1000]) \n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        for a in authors:\n",
    "            i = name_idx[a]\n",
    "            auth_years[i,0] = min(auth_years[i,0],year)\n",
    "            auth_years[i,1] = max(auth_years[i,1],year)\n",
    "    working_years = (auth_years[:,1] - auth_years[:,0]+1)\n",
    "    scoreV['working_years'] = working_years\n",
    "    scoreV['auth_years'] = auth_years\n",
    "    scoreV['last_years'] = last_years\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if PROCESS_DATA:\n",
    "    valid_ns = set()\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        valid_ns.add(n)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if USE_LIMITS:\n",
    "    for i in range(max(valid_ns)):\n",
    "        valid_ns.add(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_types = {\n",
    "        'RI': ri_confs,\n",
    "        'ML':['NIPS','ICML','AAAI','AISTATS','IJCAI','UAI','CoRL','ICLR'],\n",
    "        'CV':['CVPR','ICCV','ECCV','IEEE Trans. Pattern Anal. Mach. Intell.','FGR','Int. J. Comput. Vis.','WACV','BMVC','ACCV'],\n",
    "        'ROB':['HRI','Int. J. Robotics Res.','Robotics: Science and Systems','Humanoids','WAFR','IROS','ICRA','FSR','ISER','ISRR','AAMAS','IEEE Robotics Autom. Lett.','IEEE Trans. Robotics and Automation'],\n",
    "        'GR':['ACM Trans. Graph.','Comput. Graph. Forum','SIGGRAPH','SIGGRAPH Asia','Symposium on Computer Animation'],\n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "am_types = ['full','1/n','1/i']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if PROCESS_DATA:\n",
    "    confTypeN = len(conf_types)+1\n",
    "    YearConf = scipy.sparse.lil_matrix((n_confs*years_per_conf,years_per_conf*confTypeN))\n",
    "    for i in range(years_per_conf):\n",
    "        year_filter = np.zeros_like(clf).reshape((-1,years_per_conf))\n",
    "        year_filter[:,i] = 1\n",
    "        YearConf[:,i*confTypeN] = (clf * year_filter.reshape(clf.shape))[:,np.newaxis]\n",
    "        j = 1\n",
    "        for f_type, f_confs in conf_types.items():\n",
    "            year_filter = np.zeros_like(clf).reshape((-1,years_per_conf))\n",
    "            for conf in f_confs:\n",
    "                year_filter[conf_idx[conf],i] = 1\n",
    "            YearConf[:,i*confTypeN+j] = (clf * year_filter.reshape(clf.shape))[:,np.newaxis]\n",
    "            j+=1\n",
    "    YearConf = scipy.sparse.csr_matrix(YearConf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse\n",
    "import gc\n",
    "if PROCESS_DATA:\n",
    "    for amt in am_types:\n",
    "\n",
    "        per_author_val = {}\n",
    "\n",
    "        if amt == 'full':\n",
    "            for n in valid_ns:\n",
    "                author_scores = np.ones(n)\n",
    "                per_author_val[n] = author_scores\n",
    "        elif amt == '1/n':\n",
    "            for n in valid_ns:\n",
    "                author_scores = (np.ones(n))\n",
    "                per_author_val[n] = author_scores/author_scores.sum()\n",
    "        elif amt == '1/i':\n",
    "            for n in valid_ns:\n",
    "                author_scores = 1/(np.arange(n)+1)\n",
    "                per_author_val[n] = author_scores/author_scores.sum()\n",
    "        else:\n",
    "            raise\n",
    "\n",
    "        count_vecs = {}\n",
    "        paper_vecs = []\n",
    "        for paper in all_papers:\n",
    "                \n",
    "            tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "            n = len(authors)\n",
    "            j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "\n",
    "            author_scores = per_author_val[n]\n",
    "            if USE_LIMITS:\n",
    "                tmpp = []\n",
    "                tmpapers = authors[:-1] if n >= 2 else authors\n",
    "                tmpscores = author_scores[:-1] if n >= 2 else author_scores\n",
    "\n",
    "                for a,v in zip(tmpapers,tmpscores):\n",
    "                    idx = name_idx[a]\n",
    "                    if year > last_years[idx]:\n",
    "                        continue\n",
    "                    tmpp.append((idx,j,v))\n",
    "                paper_vecs.append(tmpp)\n",
    "            else:\n",
    "                paper_vecs.append([(name_idx[a],j,v) for a,v in zip(authors,author_scores)])\n",
    "\n",
    "        Xauth = scipy.sparse.dok_matrix((n_auths,years_per_conf*n_confs))\n",
    "        xdict = {}\n",
    "\n",
    "        for paper_vec in paper_vecs:\n",
    "            for i,j,v in paper_vec:\n",
    "                xdict[(i,j)] = v + xdict.get((i,j),0)\n",
    "\n",
    "        Xauth.my_update(xdict)\n",
    "\n",
    "        Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "\n",
    "\n",
    "\n",
    "        scoreV[amt] = Xauth @ YearConf\n",
    "\n",
    "\n",
    "        paper_vec = []\n",
    "        xdict = {}\n",
    "        gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if PROCESS_DATA:\n",
    "    import gzip\n",
    "    import pickle\n",
    "    with gzip.open('scoresV2.pkl.gz','wb') as fp:\n",
    "        pickle.dump(scoreV,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(scoreV),PROCESS_DATA)#,years_per_conf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.ndimage\n",
    "for am in am_types:\n",
    "    #scores = np.array(scoreV[am]).reshape((n_auths,years_per_conf,-1)).astype(np.float32)\n",
    "    scores = np.array(scoreV[am].todense()).reshape((n_auths,years_per_conf,-1)).astype(np.float32)\n",
    "    scores = np.transpose(scores,(0,2,1))\n",
    "    smooth_kernel = scipy.ndimage.gaussian_filter1d(np.identity(years_per_conf,np.float32),1)\n",
    "    scores = scores @ smooth_kernel\n",
    "    scoreV[am] = scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sTypes = ['Full'] + [k for k,v in conf_types.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.dtype,scores.nbytes,gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth_years = scoreV['auth_years']\n",
    "working_years = scoreV['working_years']\n",
    "\n",
    "total_scores = scoreV['1/i'][:,sTypes.index('Full')].sum(1)\n",
    "ri_scores = scoreV['1/i'][:,sTypes.index('RI')].sum(1)\n",
    "ri_eff_scores = ri_scores/working_years#,np.maximum(auth_years[:,1]-2000,1))\n",
    "\n",
    "ri_scores_max = scoreV['1/i'][:,sTypes.index('RI')].max(1)\n",
    "ri_scores_max_yr = np.argmax(scoreV['1/n'][:,sTypes.index('RI')],axis=1)*YEAR_BLOCKS + min_year\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#best_idx = np.argsort(total_scores)[::-1]\n",
    "#for k in range(10):\n",
    "#    idx = best_idx[k]\n",
    "#    print('{:30s}\\t{:.2f}'.format(all_authors[idx],total_scores[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interesting_set = set()\n",
    "with open('top_ri_Metric.json','rt') as fp:\n",
    "    interesting_set = set(json.load(fp))\n",
    "    print(len(interesting_set))\n",
    "#interesting_set.add('Jeff Clune')\n",
    "interesting_set = set([_ for _ in uni_names if _ in name_idx])\n",
    "print(len(interesting_set))\n",
    "interesting_set.add('Dinesh Jayaraman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pot_cand_df = pd.read_csv('pot_export.csv',index_col=0)\n",
    "#interesting_set = set(pot_cand_df.Author)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ri_cand_names.pkl','rb') as fp:\n",
    "    profile_set = set(    pickle.load(fp))\n",
    "profile_set.remove('David Rosen')\n",
    "profile_set.add('David M. Rosen')\n",
    "profile_set.add('Vaggos Chatziafratis')\n",
    "\n",
    "interesting_set = set(list(profile_set)).union(prev_cand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Directors = ['Takeo Kanade','Martial Hebert','Matthew T. Mason']\n",
    "labels = ['TK','MH','MM','RD']\n",
    "for n in Directors:\n",
    "    interesting_set.add(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(ri_scores_max * (working_years > 5))[::-1]\n",
    "for k in range(1000):\n",
    "    idx = best_idx[k]\n",
    "    #interesting_set.add(all_authors[idx])\n",
    "    print('{}\\t{:30s}\\t{:.2f}\\t{:.2f}\\t{:.2f}\\t{:d}\\t{:d}'.format(k,all_authors[idx],ri_scores_max[idx],ri_eff_scores[idx],ri_scores[idx],ri_scores_max_yr[idx],int(working_years[idx])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(ri_eff_scores * (working_years > 5))[::-1]\n",
    "for k in range(150):\n",
    "    idx = best_idx[k]\n",
    "    #interesting_set.add(all_authors[idx])\n",
    "    print('{}\\t{:30s}\\t{:.2f}\\t{:.2f}\\t{:d}'.format(k,all_authors[idx],ri_eff_scores[idx],ri_scores[idx],int(working_years[idx])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(ri_scores)[::-1]\n",
    "for k in range(150):\n",
    "    idx = best_idx[k]\n",
    "    #interesting_set.add(all_authors[idx])\n",
    "\n",
    "    print('{}\\t{:30s}\\t{:.2f}\\t{:.2f}\\t{:d}'.format(k,all_authors[idx],ri_scores[idx],ri_scores[idx]/total_scores[idx],int(auth_years[idx,0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(total_scores)[::-1]\n",
    "for k in range(len(best_idx)):\n",
    "    idx = best_idx[k]\n",
    "    #interesting_set.add(all_authors[idx])\n",
    "    if idx == 228644:\n",
    "        print('{}\\t{:30s}\\t{:.2f}\\t{:.2f}\\t{:d}'.format(k,all_authors[idx],ri_scores[idx],ri_scores[idx]/total_scores[idx],int(auth_years[idx,0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_idx['Benjamin A. Newman']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('top_ri3.json','wt') as fp:\n",
    "#    json.dump(sorted(list(interesting_set)),fp,sort_keys=True,indent=4, separators=(',', ': '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faculty_affil = pd.read_csv('faculty-affiliations.csv')\n",
    "year_span = (auth_years[:,1] - auth_years[:,0]) + 1\n",
    "faculty_lookup = {_[1]:_[2] for _ in faculty_affil.itertuples()}\n",
    "faculty_lookup['Reid G. Simmons'] = 'Carnegie Mellon University'\n",
    "faculty_lookup['Sebastian Scherer'] = 'Carnegie Mellon University'\n",
    "faculty_lookup['Jeff G. Schneider'] = 'Carnegie Mellon University'\n",
    "for row in cmu_uni.itertuples():\n",
    "    faculty_lookup[row[1]] = 'Carnegie Mellon University'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Author affiliated stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coauthors  = defaultdict(lambda: defaultdict(int))\n",
    "coauthors_frac  = defaultdict(lambda: defaultdict(float))\n",
    "coauthors_num  = defaultdict(list)\n",
    "author_pos  = defaultdict(list)\n",
    "conf_paper_frac  =  defaultdict(lambda: defaultdict(int))\n",
    "paper_num  = defaultdict(int)\n",
    "paper_frac  = defaultdict(float)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    parse = False\n",
    "    for a in authors:\n",
    "        if a in interesting_set:\n",
    "            parse = True\n",
    "            break\n",
    "    if parse:\n",
    "        for i,a in enumerate(authors):\n",
    "            if a in interesting_set:\n",
    "                conf_paper_frac[a][venue] += 1/n\n",
    "                coauthors_num[a].append(n)\n",
    "                author_pos[a].append((i+1)/n)\n",
    "                paper_num[a] += 1\n",
    "                paper_frac[a] += 1/n\n",
    "                for a2 in authors:\n",
    "                    if a2 == a:\n",
    "                        continue\n",
    "                    coauthors[a][a2] += 1\n",
    "                    coauthors_frac[a][a2] += 1/n\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted([(v,k) for k,v in coauthors['Martial Hebert'].items()],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(coauthors_num['Martial Hebert']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr_full = pickle.load(open('new_pagerank_people.pkl','rb'))\n",
    "pr_ri = pickle.load(open('new_pagerank_people_ri.pkl','rb'))\n",
    "pr_full /= pr_full.max()\n",
    "pr_ri /= pr_ri.max()\n",
    "print(pr_ri.shape,pr_full.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_names = ['Huijuan Xu','David A. B. Hyde','Elahe Soltanaghaei','Ankit Shah 0003']\n",
    "for n in remove_names:\n",
    "    if n in interesting_set:\n",
    "        interesting_set.remove(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unidecode import unidecode\n",
    "author_results = defaultdict(dict)\n",
    "for name in interesting_set:\n",
    "    idx = name_idx[name]\n",
    "    author_results[name]['Affiliation'] = faculty_lookup.get(name,'Unknown')\n",
    "    author_results[name]['Years'] = scoreV['working_years'][idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoreV['1/i_total_1970'] = scoreV['1/i'][:,sTypes.index('Full')].sum(1)\n",
    "scoreV['1/i_RI_1970'] = scoreV['1/i'][:,sTypes.index('RI')].sum(1)\n",
    "for sub in ['ROB','CV','GR','ML']:\n",
    "    den = scoreV['1/i_{}_1970'.format(sub)] = scoreV['1/i'][:,sTypes.index(sub)].sum(1)\n",
    "scoreV['full_total_1970'] = scoreV['full'][:,sTypes.index('Full')].sum(1)\n",
    "scoreV['1/n_total_1970'] = scoreV['1/n'][:,sTypes.index('Full')].sum(1)\n",
    "\n",
    "\n",
    "scoreV['1/n_max'] = scoreV['1/n'][:,sTypes.index('Full')].max(1)\n",
    "scoreV['1/n_max_yr'] = np.argmax(scoreV['1/n'][:,sTypes.index('Full')],axis=1)*YEAR_BLOCKS+min_year\n",
    "\n",
    "scoreV['1/i_max'] = scoreV['1/i'][:,sTypes.index('Full')].max(1)\n",
    "scoreV['1/i_max_yr'] = np.argmax(scoreV['1/i'][:,sTypes.index('Full')],axis=1)*YEAR_BLOCKS+min_year\n",
    "\n",
    "scoreV['full_max'] = scoreV['full'][:,sTypes.index('Full')].max(1)\n",
    "scoreV['full_max_yr'] = np.argmax(scoreV['full'][:,sTypes.index('Full')],axis=1)*YEAR_BLOCKS+min_year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in scoreV.items():\n",
    "    print(k,v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_auth_pos = np.zeros(n_auths)\n",
    "avg_auth_cnt = np.zeros(n_auths)\n",
    "auth_been_last = np.zeros(n_auths)\n",
    "auth_first_last_year = 3000*np.ones(n_auths)\n",
    "\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    if n  > 1:\n",
    "        for i,a in enumerate(authors):\n",
    "            idx = name_idx[a]\n",
    "            pos = i/(n-1)\n",
    "            auth_been_last[idx] += int(pos == 1)\n",
    "            avg_auth_pos[idx] += pos\n",
    "            avg_auth_cnt[idx] += 1\n",
    "        auth_first_last_year[name_idx[authors[-1]]] = min(year,auth_first_last_year[name_idx[authors[-1]]])\n",
    "np.save('last_years',auth_first_last_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_years = np.load('last_years.npy')\n",
    "scoreV['last_years'] = last_years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = scoreV['1/i_total_1970']/(np.maximum(2,working_years.astype(np.float32)))\n",
    "ratio_v = np.maximum(1e-3,scoreV['1/i_RI_1970'])/np.maximum(1e-3,scoreV['1/i_total_1970'])\n",
    "v *= np.sqrt(ratio_v)\n",
    "v *= 1/np.log(np.maximum(2,working_years.astype(np.float32)))\n",
    "tv = np.zeros_like(scoreV['1/i_total_1970'])\n",
    "for n in ['_apmFalse', '_apmTrue',  'pw_apmFalse', 'pw_apmTrue', 'pweff_apmFalse', 'pweff_apmTrue', 'pwunk_apmFalse','pwunk_apmTrue']:\n",
    "    print(n,scoreV[n].shape)\n",
    "    tv +=  scoreV[n]\n",
    "v *= np.log(np.maximum(tv,10))\n",
    "v *= np.log(np.maximum(np.exp(1),scoreV['full'][:,sTypes.index('Full')].sum(1)))\n",
    "v *= np.log(scoreV['1/i_max_yr']-1965)\n",
    "v = np.nan_to_num(v)\n",
    "meta_metric = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = name_idx['Maria Bauzá']\n",
    "scoreV['1/i_{}_1970'.format('CV')][idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name in interesting_set:\n",
    "    idx = name_idx[name]\n",
    "    author_results[name]['MetaMetric'] = v[idx]\n",
    "    author_results[name]['Score (1/pos)'] = scoreV['1/i_total_1970'][idx]\n",
    "    author_results[name]['RIScore'] = scoreV['1/i_RI_1970'][idx]\n",
    "    author_results[name]['RI %'] = scoreV['1/i_RI_1970'][idx]/scoreV['1/i_total_1970'][idx]\n",
    "\n",
    "    sum_v = 0\n",
    "    for sub in ['ROB','CV','GR','ML']:\n",
    "        den = scoreV['1/i_{}_1970'.format(sub)][idx]\n",
    "        den = den if den != 0.0 else 0\n",
    "        author_results[name][sub + ' %'] = den/max(1e-9,scoreV['1/i_total_1970'][idx])\n",
    "        sum_v += den\n",
    "    author_results[name]['Other %'] = max(0,scoreV['1/i_total_1970'][idx] - sum_v)/scoreV['1/i_total_1970'][idx]\n",
    "    \n",
    "    author_results[name]['From'] = scoreV['auth_years'][idx][0]\n",
    "    author_results[name]['Until'] = scoreV['auth_years'][idx][1]\n",
    "\n",
    "    author_results[name]['YearlyScore (1/pos)'] = scoreV['1/i_total_1970'][idx]/scoreV['working_years'][idx]\n",
    "    author_results[name]['YearlyRIScore'] = scoreV['1/i_RI_1970'][idx]/scoreV['working_years'][idx]\n",
    "\n",
    "new_set = set()\n",
    "\n",
    "for name in interesting_set:\n",
    "    idx = name_idx[name]\n",
    "\n",
    "    author_results[name]['avgCoauthor'] = np.array(coauthors_num[name]).mean()\n",
    "    colabs = sorted([(v,k) for k,v in coauthors_frac[name].items()],reverse=True)\n",
    "    fam_colab = sorted([(v*scoreV['1/i_total_1970'][name_idx[k]],k) for k,v in coauthors_frac[name].items()],reverse=True)\n",
    "    \n",
    "    freq_colabs = sorted([(v,k) for k,v in coauthors[name].items() if v >= 4],reverse=True)\n",
    "    if len(colabs) > 0:\n",
    "        author_results[name]['mostCoauthorName'] = unidecode(colabs[0][1])\n",
    "        author_results[name]['mostCoauthorTimes'] = colabs[0][0]\n",
    "        new_set.add(colabs[0][1])\n",
    "    else:\n",
    "        author_results[name]['mostCoauthorName'] = ''\n",
    "        author_results[name]['mostCoauthorTimes'] = 0\n",
    "\n",
    "    if len(fam_colab) > 0:\n",
    "        author_results[name]['famCoauthorName'] = unidecode(fam_colab[0][1])\n",
    "        new_set.add(fam_colab[0][1])\n",
    "    else:\n",
    "        author_results[name]['famCoauthorName'] = ''\n",
    "\n",
    "    author_results[name]['authorPosition%'] = np.array(author_pos[name]).mean()\n",
    "    author_results[name]['totalCoauth'] = len(colabs)\n",
    "    author_results[name]['freqCoauth (> 3 papers)'] = len(freq_colabs)\n",
    "    author_results[name]['famCoauthFrac'] = sum([_[0] for _ in fam_colab])\n",
    "    author_results[name]['totalCoauthFrac'] = sum([_[0] for _ in colabs])\n",
    "    \n",
    "    author_results[name]['mostPaperConf'] = sorted([(v,k) for k,v in conf_paper_frac[name].items()],reverse=True)[0][1]\n",
    "    author_results[name]['venuesPublishedIn'] = len(conf_paper_frac[name].items())\n",
    "    author_results[name]['pageRank'] = pr_full[idx]\n",
    "    #author_results[name]['pageRankRI'] = pr_ri[idx]\n",
    "    \n",
    "    \n",
    "    author_results[name]['numPapers'] = paper_num[name]\n",
    "    author_results[name]['numPapersFrac'] = paper_frac[name]\n",
    "\n",
    "\n",
    "    author_results[name]['YearlyScore (1/n)'] = scoreV['1/n_total_1970'][idx]/scoreV['working_years'][idx]\n",
    "    author_results[name]['YearlyScore (Full)'] = scoreV['full_total_1970'][idx]/scoreV['working_years'][idx]\n",
    "    author_results[name]['Score (1/n)'] = scoreV['1/n_total_1970'][idx]\n",
    "    author_results[name]['Score (Full)'] = scoreV['full_total_1970'][idx]\n",
    "    \n",
    "    author_results[name]['MaxScore'] = scoreV['1/i_max'][idx]\n",
    "    author_results[name]['MaxScore (1/n)'] = scoreV['1/n_max'][idx]\n",
    "    author_results[name]['MaxScore (Full)'] = scoreV['full_max'][idx]\n",
    "    \n",
    "    author_results[name]['BestYear'] = scoreV['1/i_max_yr'][idx]\n",
    "    author_results[name]['BestYear (1/n)'] = scoreV['1/n_max_yr'][idx]\n",
    "    author_results[name]['BestYear (Full)'] = scoreV['full_max_yr'][idx]\n",
    "    \n",
    "    for i,n in enumerate(['_apmFalse', '_apmTrue',  'pw_apmFalse', 'pw_apmTrue', 'pweff_apmFalse', 'pweff_apmTrue', 'pwunk_apmFalse','pwunk_apmTrue']):\n",
    "        author_results[name]['Adv'+str(i+1)] =  scoreV[n][idx]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_list = []\n",
    "for k,v in author_results.items():\n",
    "    v['Name'] = k\n",
    "    results_list.append(v)\n",
    "def_order = list(author_results[list(author_results.keys())[0]].keys())\n",
    "df_results = pd.DataFrame(results_list)[def_order].set_index('Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results = df_results.fillna(0.0)\n",
    "df_results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "ss = StandardScaler()\n",
    "vecs = ss.fit_transform(df_results._get_numeric_data())\n",
    "pca = PCA(n_components=5,whiten=True)\n",
    "res = pca.fit_transform(vecs)\n",
    "pca.explained_variance_\n",
    "for i in range(5):\n",
    "    df_results['pca'+str(i)] = res[:,i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_out = df_results[~df_results.index.isin(Directors)]\n",
    "pot_name = pot_cand_df.set_index('Author')\n",
    "pot_name.index = pot_name.index.rename('Name')\n",
    "df_results_out2= df_results_out.copy()\n",
    "#df_results_out2 = df_results_out.join(pot_name[pot_name.columns.difference(df_results_out.columns)])\n",
    "adv_totals = df_results_out2[[_ for _ in df_results_out2.columns if 'Adv' in _ and 'NSF' not in _]].sum(1)\n",
    "adv_min = df_results_out2[[_ for _ in df_results_out2.columns if 'Adv' in _ and 'NSF' not in _]].min(1)\n",
    "adv_max = df_results_out2[[_ for _ in df_results_out2.columns if 'Adv' in _ and 'NSF' not in _]].max(1)\n",
    "\n",
    "df_hits = pd.read_excel('google hits.xlsx')\n",
    "df_hits= df_hits.set_index('Name')\n",
    "df_results_out2 = df_results_out2.join(df_hits)\n",
    "\n",
    "df_results_out2['AdvMin'] = adv_min\n",
    "df_results_out2['AdvMax'] = adv_max\n",
    "df_results_out2['AdvTotal'] = adv_totals\n",
    "df_results_out.shape,df_results_out2.shape\n",
    "df_results_out2['hits'] = np.log(1+df_results_out2['hits'])\n",
    "\n",
    "with open('s2.pkl','rb') as fp:\n",
    "    s2 = pickle.load(fp)\n",
    "    s2['David M. Rosen'] = s2['David Rosen']\n",
    "s2r = {}\n",
    "for n in df_results_out2.index:\n",
    "    s2r[n] =     {k: np.log(1+s2[n][k]) for k in ['iTot','iInf','tTot','tInf']}\n",
    "df_results_out2 = df_results_out2.join(pd.DataFrame(s2r).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_v = df_results[df_results.index.isin(profile_set)]['Score (1/pos)']\n",
    "#rev_v = prev_v[1:]\n",
    "prev_v.mean(), prev_v.std(),prev_v.max(),prev_v.min()\n",
    "\n",
    "#df_results.loc['Maggie Wigness']\n",
    "#prev_v.shape\n",
    "#((prev_v > 15) &(prev_v < 25)).sum()\n",
    "#df_results[[_ for _ in df_results.columns if 'score' in _.lower()]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_out2['MetaMetric'] = df_results_out2['YearlyRIScore']  + np.sqrt(df_results_out2['YearlyScore (1/pos)'])\n",
    "df_results_out2['MetaMetric'] = df_results_out2['MetaMetric'] + 0.1* (df_results_out2['Score (1/pos)']-df_results_out2['RIScore'])\n",
    "df_results_out2['MetaMetric'] = df_results_out2['MetaMetric'] + 0.1 * adv_totals\n",
    "\n",
    "vecL = []\n",
    "for row in df_results_out2[['ROB %','CV %', 'GR %', 'ML %', 'Other %']].itertuples():\n",
    "    vec = [_ for _ in row[1:] if _ > 0]\n",
    "    vecL.append(np.std(vec) if len(vec) > 1 else 0)\n",
    "    \n",
    "df_results_out2['Var'] = vecL\n",
    "df_results_out2['Words'] = [word_person[a] for a in df_results_out2.index]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_out2 = df_results_out2.sort_values('MetaMetric',0,False)\n",
    "#df_results_out2[[_ for _ in df_results_out2.columns if '%' in _ and 'author' not in _ and 'Other' not in _]].sum(1)\n",
    "df_results_out2#[df_results_out2.MetaMetric > 0]\n",
    "#df_out2 = df_results_out2[(df_results_out2['RI %'] > 0.1) & (df_results_out2.MetaMetric > 0) & (df_results_out2['Score (1/pos)'] > 5)]\n",
    "#.loc['Júlia Borràs Sol']\n",
    "\n",
    "numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "newdf = df_results_out2.select_dtypes(include=numerics)\n",
    "df_prev_cand = newdf[newdf.index.isin(prev_cand)]\n",
    "\n",
    "normed_df = 10*(newdf-df_prev_cand.mean())/df_prev_cand.std()+50\n",
    "normed_df['Prod (I)'] = normed_df['Score (1/pos)']\n",
    "normed_df['Prod (T)'] = normed_df['Score (Full)']\n",
    "normed_df['Cite (I)'] = normed_df['iTot']\n",
    "normed_df['Cite (T)'] = normed_df['tTot']\n",
    "normed_df['CiteI (I)'] = normed_df['iInf']\n",
    "normed_df['CiteI (T)'] = normed_df['tInf']\n",
    "\n",
    "normed_df['Fit'] = normed_df['RI %']\n",
    "normed_df['Unique'] = 100-normed_df['Words']\n",
    "normed_df['News'] = normed_df['hits']\n",
    "normed_df['Network'] = normed_df['pageRank']\n",
    "normed_df['Variance'] = normed_df['Var']\n",
    "\n",
    "w = np.array([60,20,7,3,7,3,20,10,20,5,1])\n",
    "#w = np.array([10., 10.,  9.,  8.,  7.,  7., 19.,  3., 17.,  8.,  3.])\n",
    "#w = np.array([10., 10.,  7.,  7.,  6.,  6., 19.,  4., 20.,  8.,  4.])\n",
    "#w = np.array([13.,  5.,  1., 13., 18.,  0., 11.,  8., 14.,  9.,  8.])\n",
    "w = w/w.sum()\n",
    "print(w)\n",
    "normed_df['Est'] = sum([wt*np.maximum(20,normed_df[col]) for wt,col in zip(w,['Prod (I)','Prod (T)','Cite (I)','Cite (T)','CiteI (I)','CiteI (T)','Fit','Unique','News','Network','Variance'])])\n",
    "\n",
    "out_df = normed_df.iloc[:,-12:].sort_values('Est',0,False)\n",
    "df_results_out2['MetaMetric'] = out_df['Est']\n",
    "df_results_out2 = df_results_out2.sort_values('MetaMetric',0,False)\n",
    "\n",
    "#out_df = out_df[out_df.index.isin(profile_set)]\n",
    "out_df.round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "#clf = sm.Logit(np.array('Carnegie' in df_results_out2.Affiliation).astype(int),np.array(out_df)[:,:-1]).fit()\n",
    "#clf\n",
    "yv = ('Carnegie Mellon University'== np.array(df_results_out2.Affiliation)).astype(int)\n",
    "xv = out_df[[c for c in out_df.columns if c != 'Est']]\n",
    "clf= sm.Logit(yv,xv).fit()\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression,LogisticRegressionCV\n",
    "clf.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clf =LogisticRegressionCV(25,max_iter=1e3,class_weight='balanced').fit(xv,yv)\n",
    "#clf =LogisticRegression(C=1e-3,max_iter=1e3).fit(xv,yv)\n",
    "\n",
    "w = clf.params\n",
    "w -= np.min(w)\n",
    "w = 100*w.ravel()/w.sum()\n",
    "w.round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_results_out2['Pred'] = clf.predict(xv)\n",
    "#df_results_out2=df_results_out2.sort_values('Pred')\n",
    "#df_results_out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(w * 100).round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "out_df[out_df.index.isin(prev_cand)].round(0).astype(int).sort_values('Est',0,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 150)\n",
    "out_df[out_df.index.isin(profile_set)].round(0).astype(int).sort_values('Est',0,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results_out2.loc[df_results_out2.index.isin(prev_cand),'Affiliation'] = 'Carnegie Mellon University'\n",
    "df_results_out2.join(out_df).to_excel('filter_names.xlsx')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "newdf.mean(),newdf.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('s2.pkl','rb') as fp:\n",
    "    s2 = pickle.load(fp)\n",
    "with open('s2p.pkl','rb') as fp:\n",
    "    s2p = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,p in s2p.items():\n",
    "    break\n",
    "dfs2p = pd.DataFrame(s2p).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#(np.array(v.mean())+np.array(v.std())).shape\n",
    "#dfs2p= pd.to_numeric(dfs2p.stack(), errors='coerce').unstack()\n",
    "p['citations'][0],p['year'],len(p['references'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs2p['sc'] = dfs2p['citations'].map(lambda x: len(x) if type(x) == list else 0)\n",
    "dfs2p['sc'] = dfs2p['sc']+1\n",
    "v = dfs2p[['year','sc']].groupby('year')\n",
    "plt.plot(v.mean().index,v.mean())\n",
    "#plt.fill_between(v.mean().index,np.array(v.mean())[:,0]+np.array(v.std())[:,0],np.array(v.mean())[:,0]-np.array(v.std())[:,0],alpha=0.5)\n",
    "plt.xlim(2010,2021)\n",
    "plt.ylim(0,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.optimize as opt\n",
    "baseline = np.array(v.mean().iloc[-8:])[:,0]\n",
    "def curve_fit(x):\n",
    "    x[1] = np.clip(x[1],0,1e9)\n",
    "    vec = np.arange(-7,1)\n",
    "    #vres = np.log(-x[1]*vec+1+x[0])\n",
    "    #vres = (1.0-1/(1+np.exp(-vec*x[1])))*x[0]\n",
    "    vres = np.tanh(-vec*x[1]+x[2])*x[0]\n",
    "    return np.linalg.norm(baseline-vres)\n",
    "res = opt.minimize(curve_fit,np.array([1,1,1]))\n",
    "\n",
    "vec = np.arange(-7,1)\n",
    "plt.plot(vec[1:],baseline[1:])\n",
    "vec = np.arange(-20,1)\n",
    "cite_corr = lambda x: np.tanh(-x*0.338-0.1436)*52.49\n",
    "vres = cite_corr(vec)\n",
    "plt.plot(vec,vres)\n",
    "plt.xlim(-1,-20)\n",
    "\n",
    "res.x,res.fun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "curr_year = dfs2p.year.max()\n",
    "def cite_correct(year,cite):\n",
    "    if year == curr_year:\n",
    "        year = curr_year-1\n",
    "    if year is None:\n",
    "        year = curr_year - 10\n",
    "    curr = cite_corr(year-curr_year)\n",
    "    teny = cite_corr(10-curr_year)\n",
    "    return cite*teny/curr\n",
    "cite_corr(-1),cite_corr(-20),cite_corr(-10),cite_corr(-5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in s2p:\n",
    "    year = s2p[k]['year'] if 'year' in s2p[k] and s2p[k]['year'] is not None else curr_year\n",
    "    cite = len(s2p[k]['citations'] if 'citations' in s2p[k] else [])\n",
    "    s2p[k]['sc'] = cite_correct(year,cite)\n",
    "    cite = len([1 for _ in s2p[k]['citations'] if _['isInfluential']]if 'citations' in s2p[k] else [])\n",
    "    s2p[k]['sci'] = cite_correct(year,cite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2p[k]['sci'],s2p[k]['sc'],len(s2p[k]['citations'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in s2:\n",
    "    # 4 things\n",
    "    d = {\n",
    "        'tInf':0,\n",
    "        'iInf':0,\n",
    "        'tTot':0,\n",
    "        'iTot':0,\n",
    "    }\n",
    "    aid = s2[a]['authorId']\n",
    "    if type(s2[a]['authorId']) == str:\n",
    "        aid = [int(s2[a]['authorId'])]\n",
    "    aid = [int(_) for _ in aid]\n",
    "    for p in s2[a]['papers']:\n",
    "        paper = s2p[p['paperId']]\n",
    "        if 'fieldsOfStudy' in paper and type(paper['fieldsOfStudy']) == list and len(paper['fieldsOfStudy']) > 0 and sum([k in paper['fieldsOfStudy'] for k in ['Materials Science','Computer Science','Engineering','Medicine','Psychology','Mathematics']]) == 0:\n",
    "            continue\n",
    "        d['tTot'] += paper['sc']\n",
    "        d['tInf'] += paper['sci']\n",
    "        \n",
    "        tot = 0\n",
    "        val = 0\n",
    "        for i,n in enumerate(paper['authors']):\n",
    "            tot += 1/(1+i)\n",
    "            if n['authorId'] != None and int(n.get('authorId',0)) in aid:\n",
    "                val = 1/(1+i)\n",
    "        mul = (val/tot)\n",
    "        d['iTot'] += mul*paper['sc']\n",
    "        d['iInf'] += mul*paper['sci']\n",
    "    s2[a].update(d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'fieldsOfStudy' in paper , type(paper['fieldsOfStudy']) == list , len(paper['fieldsOfStudy']) > 0 and sum([k in paper['fieldsOfStudy'] for k in ['Computer Science','Engineering','Medicine','Psychology','Mathematics']])\n",
    "#with open('s2.pkl','wb') as fp:\n",
    "#    pickle.dump(s2,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import re\n",
    "import time\n",
    "browser = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    import copy\n",
    "    for n in list(df_results_out2.index)[len(res):]:\n",
    "        on = copy.deepcopy(n)\n",
    "        if n.split()[-1][:2] == '00':\n",
    "            n = ' '.join(n.split()[:-1])\n",
    "        browser.get('https://www.semanticscholar.org/search?q={}&sort=relevance'.format(n))\n",
    "\n",
    "        element = WebDriverWait(browser, 10).until(\n",
    "            EC.presence_of_element_located((By.CLASS_NAME, \"matched-author-shoveler__author-title\"))\n",
    "        )\n",
    "        txt = browser.page_source\n",
    "        ids = re.findall('<a class=\"matched-author-shoveler__author-link\" href=\"(\\/author\\/.*?)\\/([0-9]+)\">',txt)\n",
    "        res[on] = ids\n",
    "        #if len(ids) > 1:\n",
    "        #    time.sleep(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resd = []\n",
    "for k,v in res.items():\n",
    "    d = {'name':k,'ids': v[0][1], 'multi':len(v)}\n",
    "    resd.append(d)\n",
    "#pd.DataFrame(resd).set_index('name').to_csv('s2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs2 = pd.read_csv('s2.csv').set_index('name')\n",
    "\n",
    "done_set = set(sum([v['authorId'] if type(v['authorId'])==list else [v['authorId']] for k,v in s2.items()],[]))\n",
    "done_set = set([int(x) for x in done_set])\n",
    "\n",
    "proc_set = []\n",
    "for n in dfs2.itertuples():\n",
    "    t = eval(n[1])\n",
    "    if type(t) == int:\n",
    "        t = [t]\n",
    "    for i in t:\n",
    "        if i not in done_set:\n",
    "            proc_set.append((n[0],t))\n",
    "            break\n",
    "proc_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "for row in proc_set:\n",
    "    res = []\n",
    "    for i in row[1]:\n",
    "        resp = requests.get('https://api.semanticscholar.org/v1/author/{}'.format(i))\n",
    "        time.sleep(3.1)\n",
    "        res.append(json.loads(resp.content))\n",
    "    d = res[0]\n",
    "    d['authorId'] = [int(d['authorId'])]\n",
    "    for i in range(1,len(res)):\n",
    "        d2 = res[i]\n",
    "        d['aliases'] = d.get('aliases',[]) + d2.get('aliases',[])\n",
    "        d['authorId'] = d['authorId'] + [int(d2['authorId'])]\n",
    "        d['influentialCitationCount'] += d2['influentialCitationCount']\n",
    "        d['papers'] = d['papers'] + d2['papers']\n",
    "                                                    \n",
    "    s2[row[0]] = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('s2.pkl','wb') as fp:\n",
    "#    pickle.dump(s2,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "res3 = []\n",
    "for k,v in s2.items():\n",
    "    for i in v['papers']:\n",
    "        res3.append(i['paperId'])\n",
    "res3s = set(res3)\n",
    "print(len(res3s),len([_ for _ in res3s if _ not in s2p]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in [_ for _ in res3s if _ not in s2p]:\n",
    "    resp = requests.get('https://api.semanticscholar.org/v1/paper/{}'.format(i))\n",
    "    time.sleep(3.2)\n",
    "    s2p[i] = json.loads(resp.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in s2p.items():\n",
    "    if 'error' in v or 'message' in v:\n",
    "        print(v)\n",
    "#with open('s2p.pkl','wb') as fp:\n",
    "#    pickle.dump(s2p,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_new = set()\n",
    "for name in set(list(interesting_set) +list(new_set)):\n",
    "    idx = name_idx[name]\n",
    "    if scoreV['working_years'][idx] < 10:\n",
    "        pass\n",
    "    else:\n",
    "        new_new.add(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "spec_vectors = [res[list(df_results.index).index(_)] for _ in Directors]\n",
    "spec_vectors.append(np.array(spec_vectors).mean(0))\n",
    "spec_vectors = np.array(spec_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results.iloc[np.argmax(res[:,0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(res[:,0],res[:,1])\n",
    "DIR_IDX = df_results.index.isin(Directors)\n",
    "plt.scatter(spec_vectors[:,0],spec_vectors[:,1])\n",
    "#df_results.iloc[6737]\n",
    "#np.where(res[:,0] > 7) # 2004, 6737"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import cdist,squareform\n",
    "dists = cdist(spec_vectors,res,'euclidean')\n",
    "for l,d in zip(labels,dists):\n",
    "    df_results['euc' + l] = d\n",
    "dists = cdist(spec_vectors,res,'cosine')\n",
    "for l,d in zip(labels,dists):\n",
    "    df_results['cos' + l] = d\n",
    "df_results['MetaMetric'] *= (1/np.log(np.maximum(0.07,df_results['cosRD'])*50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results.sort_values('cosRD').to_csv('profile_results5.csv')\n",
    "df_results.corr('spearman').to_csv('corr.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results.sort_values('NSF YearlyAward',0,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "name_plot = ['David P. Woodruff',\"Ryan O'Donnell\",'Anupam Gupta']\n",
    "for name in name_plot:\n",
    "    plt.plot(np.arange(min_year,max_year,YEAR_BLOCKS),scoreV['1/i'][name_idx[name],sTypes.index('Full')],label=name.split()[0])\n",
    "\n",
    "\n",
    "plt.legend()\n",
    "plt.xlim(left=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#=D3*SQRT(F3)*(1/LOG(C3))*LOG(MAX(10,SUM(AN3:AU3)))*MAX(1,LOG(AE3))*(1/LOG(MAX(0.07,BN3)*50))*LOG(AM3-1965)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interesting_set = set()\n",
    "with open('top_ri3.json','rt') as fp:\n",
    "    interesting_set = set(json.load(fp))\n",
    "    print(len(interesting_set))\n",
    "#interesting_set.add('Jeff Clune')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_v = np.argsort(meta_metric)[::-1]\n",
    "len(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(interesting_set))\n",
    "beST_N = 15000\n",
    "for i in range(beST_N):\n",
    "    idx = best_v[i]\n",
    "    if working_years[idx] >= 10:\n",
    "        interesting_set.add(all_authors[idx])\n",
    "    #print(idx,all_authors[idx],v[idx],ratio_v[idx])\n",
    "print(meta_metric[best_v[beST_N]],meta_metric[name_idx['Odest Chadwicke Jenkins']],meta_metric[name_idx['Jeff Clune']])\n",
    "print(len(interesting_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('top_ri_Metric.json','wt') as fp:\n",
    "    json.dump(sorted(list(interesting_set)),fp,sort_keys=True,indent=4, separators=(',', ': '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}