Attn_Classification_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Trainers.DatasetBC import *\n",
    "from ExperimentsBC import *\n",
    "from common_code.plotting import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_names = {\n",
    "    'sst' : 'SST',\n",
    "    'imdb' : 'IMDB',\n",
    "    'agnews' : 'AG News',\n",
    "    'tweet' : 'ADR',\n",
    "    'snli' : 'SNLI',\n",
    "    '20News_sports' : '20 News Sports',\n",
    "    'diab' : 'Diabetes',\n",
    "    'cnn' : 'CNN',\n",
    "    'anemia' : 'Anemia',\n",
    "    'babi_1' : 'bAbI 1',\n",
    "    'babi_2' : 'bAbI 2',\n",
    "    'babi_3' : 'bAbI 3'\n",
    "}  \n",
    "\n",
    "dataset_order = list(map(lambda x : dataset_names[x], \n",
    "                      ['sst', 'imdb', 'tweet', 'agnews', '20News_sports', \n",
    "                       'diab', 'anemia', 'cnn', 'babi_1', 'babi_2', 'babi_3', 'snli']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'lstm+tanh'\n",
    "import os\n",
    "models = [x for x in os.listdir('graph_outputs/CorrGL-AG_kendalltau/') if model in x]\n",
    "import pandas as pd\n",
    "dfs = {}\n",
    "for d in models :\n",
    "    df = pd.read_csv('graph_outputs/CorrGL-AG_kendalltau/' + d, index_col=0)\n",
    "    dfs[d.split('+')[0]] = df.loc['Overall']\n",
    "\n",
    "import seaborn as sns\n",
    "dfs = pd.DataFrame(dfs).transpose()\n",
    "dfs.index = dfs.index.map(lambda x : dataset_names[x])\n",
    "fig = sns.barplot(y=dfs.index, x=dfs['mean'], order=dataset_order)\n",
    "\n",
    "xmin, xmax = fig.axes.get_ylim()\n",
    "fig.axes.vlines(dfs['mean'].mean(), xmin, xmax, colors='#222222')\n",
    "sns.despine()\n",
    "plt.xlabel(\"Mean Difference between Correlations\", fontsize=25)\n",
    "plt.ylabel(\"Dataset\", fontsize=25)\n",
    "plt.ylim(xmin, xmax)\n",
    "plt.xticks(fontsize=22)\n",
    "plt.yticks(fontsize=22)\n",
    "# plt.gcf().set_size_inches(15, 15)\n",
    "plt.savefig('graph_outputs/CorrGL-AG_summary.pdf', bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "models = [x for x in os.listdir('graph_outputs/CorrStats_kendalltau/') if 'lstm+tanh' in x]\n",
    "models_avg = [x for x in os.listdir('graph_outputs/CorrStats_kendalltau/') if 'average+tanh' in x]\n",
    "\n",
    "import pandas as pd\n",
    "dfs = {}\n",
    "for d, d1 in zip(models, models_avg) :\n",
    "    df = pd.read_csv('graph_outputs/CorrStats_kendalltau/' + d, index_col=0).rename(columns=lambda x : x + '_lstm')\n",
    "    df_1 = pd.read_csv('graph_outputs/CorrStats_kendalltau/' + d1, index_col=0).rename(columns=lambda x : x + '_avg')\n",
    "    df = pd.concat([df, df_1], axis=1)\n",
    "    dfs[d.split('+')[0]] = df.loc['ag']\n",
    "\n",
    "import seaborn as sns\n",
    "dfs = pd.DataFrame(dfs).transpose()\n",
    "dfs.index = dfs.index.map(lambda x : dataset_names[x])\n",
    "fig = sns.barplot(y=dfs.index, x=dfs['mean_avg'] - dfs['mean_lstm'], order=dataset_order)\n",
    "\n",
    "xmin, xmax = fig.axes.get_ylim()\n",
    "fig.axes.vlines((dfs['mean_avg'] - dfs['mean_lstm']).mean(), xmin + 0.5, xmax - 0.5, colors='#222222')\n",
    "sns.despine()\n",
    "plt.xlabel(\"Mean Difference between Correlations\", fontsize=25)\n",
    "plt.ylabel(\"Dataset\", fontsize=25)\n",
    "plt.ylim(xmin, xmax)\n",
    "plt.xticks(fontsize=22)\n",
    "plt.yticks(fontsize=22)\n",
    "# plt.savefig('graph_outputs/CorrAL(avg-lstm)_summary.pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Trainers.DatasetQA import *\n",
    "from ExperimentsQA import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets :\n",
    "    print(k)\n",
    "    dataset = datasets[k]()\n",
    "    dataset.basepath = 'outputs_dev'\n",
    "    generate_graphs_on_encoders(dataset, ['cnn', 'lstm', 'average', 'cnn_dot', 'lstm_dot', 'average_dot'])\n",
    "    print('+'*700)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datasets_ehr['mortality']()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Trainers.PlottingBC import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = run_evaluator_on_latest_model(dataset, config='lstm')\n",
    "logodds_results = pload(evaluator.model, 'logodds_attention')\n",
    "emax_jds, emax_adv_attn, emax_ad_y = plot_attn_diff(dataset, dataset.test_data, logodds_results, \n",
    "                                                    save_name='logodds_subs', dirname=evaluator.model.dirname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 308\n",
    "a = print_adversarial_example(dataset.vec.map2words(dataset.test_data.X[n]), dataset.test_data.attn_hat[n], emax_adv_attn[n], latex=True)\n",
    "dataset.test_data.yt_hat[n], emax_ad_y[n]\n",
    "print(a[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ent = [np.max(x) for x in emax_adv_attn]\n",
    "plt.scatter(ent, emax_jds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_1 = set(np.where(np.array(ent) > 0.6)[0]) & set(np.where(np.array(emax_jds) > 0.4)[0])\n",
    "idx_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset(dataset, 'logodds_lstm_reg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_logodds_experiment(dataset, 'logodds_lstm_reg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiments_on_latest_model(dataset, 'logodds_lstm_reg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_graphs_on_latest_model(dataset, 'logodds_lstm_reg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_logodds_substitution_experiment(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = dataset.test_data\n",
    "plt.scatter(test_data.yt_hat[:, 0], test_data.opp_yt_hat[:, 0], s=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff = np.abs(test_data.yt_hat[:, 0] - test_data.opp_yt_hat[:, 0])\n",
    "np.argsort(diff)[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(diff, bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = dataset.test_data\n",
    "for k, v in test_data.logodds_combined[0].items() :\n",
    "    print(dataset.vec.map2words(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 4231\n",
    "true_X = dataset.vec.map2words(test_data.X[n])\n",
    "new_X = dataset.vec.map2words(test_data.opp_X[n])\n",
    "print_attn(true_X, dataset.test_data.attn_hat[n])\n",
    "print(test_data.yt_hat[n])\n",
    "print_attn(new_X, dataset.test_data.opp_attn[n])\n",
    "print(test_data.opp_yt_hat[n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    generate_graphs_on_latest_model(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    train_dataset(dataset, 'logodds_lstm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    run_evaluator_on_latest_model(dataset, 'logodds_lstm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.LR import LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    if k != 'pheno' : continue\n",
    "    dataset = datasets_ehr[k]()\n",
    "    train_lr_on_dataset(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    push_all_models(dataset, dataset.keys_to_use)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    run_logodds_experiment(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(enumerate(dataset.vec.label_headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_top_words(dataset, config='lstm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array(dataset.test_data.yt_hat)\n",
    "idx_y = np.where(y > 0.8)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "top_words_dict = defaultdict(float)\n",
    "for i in idx_y :\n",
    "    d = dataset.test_data.top_words_attn[i]\n",
    "    for k, v in d.items() :\n",
    "        top_words_dict[k] += v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words = dict(sorted(top_words_dict.items(), key=lambda x: x[1])[-20:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.print_all_features(n=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words_attn = set(top_words.keys())\n",
    "top_words_lr = set(lr.get_features(n=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words_attn & top_words_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(top_words_attn & top_words_lr) / len(top_words_attn | top_words_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.keys_to_use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multi Adversarial Examples\n",
    "=========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "#     generate_adversarial_examples(dataset, config='lstm')\n",
    "#     generate_logodds_examples(dataset, config='lstm')\n",
    "#     generate_graphs_on_latest_model(dataset, config='lstm')\n",
    "    push_all_models(dataset, keys=dataset.keys_to_use)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.vec.label_headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in datasets_ehr :\n",
    "    dataset = datasets_ehr[k]()\n",
    "    dataset.display_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataset.test_data.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = run_evaluator_on_latest_model(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_adversarial_outputs = pload(evaluator.model, 'multi_adversarial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Trainers.PlottingBC import *\n",
    "test_data = dataset.test_data\n",
    "emax_jds, emax_adv_attn, emax_ad_y = plot_multi_adversarial(test_data.X, test_data.yt_hat, \n",
    "                                                            test_data.attn_hat, multi_adversarial_outputs, dirname=\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_adversarial_examples(dataset, test_data.X, test_data.yt_hat, test_data.attn_hat, \n",
    "                           emax_jds, emax_adv_attn, emax_ad_y, by_class=None, dirname='.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}