threshold_calc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyzing results on NIH datasets\n",
    "\n",
    "# Import necessary packages\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from balancersV2 import BinaryBalancer\n",
    "import updated_tools\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix\n",
    "import random\n",
    "random.seed(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that uses validation df to calculate threshold that maximizes Youdens-Statistic\n",
    "def get_optimal_j(df):\n",
    "    thresholds = np.linspace(0,1,501)\n",
    "    j_list = []\n",
    "    y = df['y'].values\n",
    "    for t in thresholds:\n",
    "        y_hat = (df['y_prob'].values >= t)*1\n",
    "        tn, fp, fn, tp = confusion_matrix(y, y_hat).ravel()\n",
    "        tpr = tp/(tp+fn)\n",
    "        fpr = fp/(fp+tn)\n",
    "        j_list.append(tpr-fpr)\n",
    "    return thresholds[np.argmax(np.array(j_list))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Counting fairness\n",
    "# Set relevant variables\n",
    "folder_name = 'reorganized_nih_results'\n",
    "columns = ['0/100 f', '0/100 m', '25/75 f', '25/75 m', '50/50 f', '50/50 m', '75/25 f', '75/25 m', '100/0 f', '100/0 m']\n",
    "models = ['train100%_female_images','train75%_female_images','train50%_female_images','train25%_female_images','train0%_female_images']\n",
    "condition_names = ['Atelectasis','Cardiomegaly','Effusion','Infiltration','Mass',\n",
    "                   'Nodule','Pneumonia','Pneumothorax','Consolidation','Edema',\n",
    "                   'Emphysema','Fibrosis','Pleural_Thickening','Hernia']\n",
    "num_folds = 20\n",
    "num_list = [i for i in range(num_folds)]\n",
    "fold_list = ['Fold_' + str(k) for k in range(num_folds)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train100%_female_images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [14:53<00:00, 63.80s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train75%_female_images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [16:14<00:00, 69.64s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train50%_female_images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [15:05<00:00, 64.71s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train25%_female_images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [14:51<00:00, 63.65s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train0%_female_images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [15:16<00:00, 65.48s/it]\n"
     ]
    }
   ],
   "source": [
    "# Calculate optimal thresholds\n",
    "for i, model in enumerate(models):\n",
    "    print(model)\n",
    "    favor_m = []\n",
    "    favor_f = []\n",
    "    for condition in tqdm(condition_names):\n",
    "        bias_tpr = []\n",
    "        bias_fpr = []\n",
    "        df_list = []\n",
    "        j_list = []\n",
    "        for k in range(num_folds):\n",
    "            folder_path = os.path.join(folder_name, model, condition)\n",
    "            df = pd.read_csv(os.path.join(folder_path, fold_list[k]), index_col=[0])\n",
    "            df_list.append(df)\n",
    "        for k in range(num_folds):\n",
    "            i = random.sample(num_list[:k] + num_list[k:],1)[0]\n",
    "            val = df_list[i]\n",
    "            j_list.append(get_optimal_j(val))\n",
    "        dict = {'Fold':fold_list,'Optimal Threshold':j_list}\n",
    "        pd.DataFrame(dict).to_csv(os.path.join(folder_path, 'optimal_thresholds.csv'), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('fairness_project')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "32ad6245715914d03a8b5beed738b5748673d1a133a5395eb58474bca4b35574"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}