sample_rare_classes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "423825d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b0606f8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Constants:\n",
    "    DATA_DIR = \"./tum-ai-hackathon-model-implementation/data\"\n",
    "    FULL_DIR = \"./tum-ai-hackathon-model-implementation/full_data\"\n",
    "    SUBSET_NAME=\"Dataset_Chest_X_Ray_Sample.csv\"\n",
    "    FULLSET_NAME=\"Data_Entry_2017.csv\"\n",
    "    IMG_DIR = \"images_sample\"\n",
    "    FULLIMG_DIR = \"images\"\n",
    "    IMG = \"image\"\n",
    "    LABEL = \"label\"\n",
    "    AGE = \"Patient_Age\"\n",
    "    GENDER = \"Patient_Gender\"\n",
    "    \n",
    "const = Constants()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4c8c6778",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>label</th>\n",
       "      <th>Follow_up</th>\n",
       "      <th>Patient_ID</th>\n",
       "      <th>Patient_Age</th>\n",
       "      <th>Patient_Gender</th>\n",
       "      <th>View_Position</th>\n",
       "      <th>Original_Image_Width</th>\n",
       "      <th>Original_Image_Height</th>\n",
       "      <th>OriginalImagePixelSpacing_x</th>\n",
       "      <th>OriginalImagePixelSpacing_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000005_000.png</td>\n",
       "      <td>No Finding</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>69</td>\n",
       "      <td>F</td>\n",
       "      <td>PA</td>\n",
       "      <td>2048</td>\n",
       "      <td>2500</td>\n",
       "      <td>0,168</td>\n",
       "      <td>0,168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00000005_001.png</td>\n",
       "      <td>No Finding</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>69</td>\n",
       "      <td>F</td>\n",
       "      <td>AP</td>\n",
       "      <td>2500</td>\n",
       "      <td>2048</td>\n",
       "      <td>0,168</td>\n",
       "      <td>0,168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00000005_003.png</td>\n",
       "      <td>No Finding</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>69</td>\n",
       "      <td>F</td>\n",
       "      <td>PA</td>\n",
       "      <td>2992</td>\n",
       "      <td>2991</td>\n",
       "      <td>0,143</td>\n",
       "      <td>0,143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              image       label  Follow_up  Patient_ID  Patient_Age  \\\n",
       "0  00000005_000.png  No Finding          0           5           69   \n",
       "1  00000005_001.png  No Finding          1           5           69   \n",
       "2  00000005_003.png  No Finding          3           5           69   \n",
       "\n",
       "  Patient_Gender View_Position  Original_Image_Width  Original_Image_Height  \\\n",
       "0              F            PA                  2048                   2500   \n",
       "1              F            AP                  2500                   2048   \n",
       "2              F            PA                  2992                   2991   \n",
       "\n",
       "  OriginalImagePixelSpacing_x OriginalImagePixelSpacing_y  \n",
       "0                       0,168                       0,168  \n",
       "1                       0,168                       0,168  \n",
       "2                       0,143                       0,143  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pd.read_csv(os.path.join(const.DATA_DIR, const.SUBSET_NAME), sep=';')\n",
    "dataset.rename(columns={\"Image_Index\": \"image\", \"Finding_Labels\": \"label\"}, inplace=True)\n",
    "# dataset['image'] = dataset['image'].apply(lambda x: os.path.join(const.DATA_DIR, const.IMG_DIR, x))\n",
    "dataset.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "02b8f97d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No Finding            6621\n",
       "Infiltration          1053\n",
       "Atelectasis            510\n",
       "Effusion               453\n",
       "Nodule                 292\n",
       "Pneumothorax           231\n",
       "Mass                   207\n",
       "Consolidation          141\n",
       "Pleural_Thickening     119\n",
       "Cardiomegaly           113\n",
       "Emphysema               88\n",
       "Fibrosis                87\n",
       "Edema                   55\n",
       "Pneumonia               36\n",
       "Hernia                  10\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "16570d8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>label</th>\n",
       "      <th>Follow-up #</th>\n",
       "      <th>Patient ID</th>\n",
       "      <th>Patient Age</th>\n",
       "      <th>Patient Gender</th>\n",
       "      <th>View Position</th>\n",
       "      <th>OriginalImage[Width</th>\n",
       "      <th>Height]</th>\n",
       "      <th>OriginalImagePixelSpacing[x</th>\n",
       "      <th>y]</th>\n",
       "      <th>Unnamed: 11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000001_000.png</td>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>M</td>\n",
       "      <td>PA</td>\n",
       "      <td>2682</td>\n",
       "      <td>2749</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.143</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00000001_001.png</td>\n",
       "      <td>Cardiomegaly|Emphysema</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>M</td>\n",
       "      <td>PA</td>\n",
       "      <td>2894</td>\n",
       "      <td>2729</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.143</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00000001_002.png</td>\n",
       "      <td>Cardiomegaly|Effusion</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>M</td>\n",
       "      <td>PA</td>\n",
       "      <td>2500</td>\n",
       "      <td>2048</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.168</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              image                   label  Follow-up #  Patient ID  \\\n",
       "0  00000001_000.png            Cardiomegaly            0           1   \n",
       "1  00000001_001.png  Cardiomegaly|Emphysema            1           1   \n",
       "2  00000001_002.png   Cardiomegaly|Effusion            2           1   \n",
       "\n",
       "   Patient Age Patient Gender View Position  OriginalImage[Width  Height]  \\\n",
       "0           58              M            PA                 2682     2749   \n",
       "1           58              M            PA                 2894     2729   \n",
       "2           58              M            PA                 2500     2048   \n",
       "\n",
       "   OriginalImagePixelSpacing[x     y]  Unnamed: 11  \n",
       "0                        0.143  0.143          NaN  \n",
       "1                        0.143  0.143          NaN  \n",
       "2                        0.168  0.168          NaN  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_dataset = pd.read_csv(os.path.join(const.FULL_DIR, const.FULLSET_NAME), sep=',')\n",
    "full_dataset.rename(columns={\"Image Index\": \"image\", \"Finding Labels\": \"label\"}, inplace=True)\n",
    "# full_dataset['image'] = full_dataset['image'].apply(lambda x: os.path.join(const.FULL_DIR, const.FULLIMG_DIR, x))\n",
    "full_dataset.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a892031d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No Finding                                                   60361\n",
       "Infiltration                                                  9547\n",
       "Atelectasis                                                   4215\n",
       "Effusion                                                      3955\n",
       "Nodule                                                        2705\n",
       "                                                             ...  \n",
       "Edema|Effusion|Mass|Pneumothorax                                 1\n",
       "Atelectasis|Consolidation|Effusion|Infiltration|Pneumonia        1\n",
       "Hernia|Infiltration|Nodule                                       1\n",
       "Edema|Infiltration|Mass|Pneumothorax                             1\n",
       "Consolidation|Edema|Pneumothorax                                 1\n",
       "Name: label, Length: 836, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_dataset['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "deb3a56e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No Finding            60361\n",
       "Infiltration           9547\n",
       "Atelectasis            4215\n",
       "Effusion               3955\n",
       "Nodule                 2705\n",
       "Pneumothorax           2194\n",
       "Mass                   2139\n",
       "Consolidation          1310\n",
       "Pleural_Thickening     1126\n",
       "Cardiomegaly           1093\n",
       "Emphysema               892\n",
       "Fibrosis                727\n",
       "Edema                   628\n",
       "Pneumonia               322\n",
       "Hernia                  110\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset = full_dataset[~full_dataset['label'].str.contains('\\|')]\n",
    "subset['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "93e3e494",
   "metadata": {},
   "outputs": [],
   "source": [
    "take_all = ['Hernia', 'Pneumonia', 'Edema', 'Fibrosis', \n",
    "            'Emphysema', 'Cardiomegaly', 'Pleural_Thickening', 'Consolidation', 'Mass', 'Pneumothorax']\n",
    "take_half = ['Nodule', 'Effusion', 'Atelectasis']\n",
    "take_quarter = ['Infiltration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ab30368c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No Finding            3310\n",
       "Infiltration          2387\n",
       "Pneumothorax          2194\n",
       "Mass                  2139\n",
       "Atelectasis           2108\n",
       "Effusion              1978\n",
       "Nodule                1352\n",
       "Consolidation         1310\n",
       "Pleural_Thickening    1126\n",
       "Cardiomegaly          1093\n",
       "Emphysema              892\n",
       "Fibrosis               727\n",
       "Edema                  628\n",
       "Pneumonia              322\n",
       "Hernia                 110\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_dfs = []\n",
    "subset_dfs.append(full_dataset[full_dataset['label'].isin(take_all)])\n",
    "\n",
    "for lab in take_half:\n",
    "    subset_dfs.append(full_dataset[full_dataset['label'] == lab].sample(frac=0.5))\n",
    "\n",
    "for lab in take_quarter:\n",
    "    subset_dfs.append(full_dataset[full_dataset['label'] == lab].sample(frac=0.25))\n",
    "    \n",
    "subset_dfs.append(dataset[dataset['label'] == 'No Finding'].sample(frac=0.5))\n",
    "\n",
    "sampled_df = pd.concat(subset_dfs)\n",
    "sampled_df['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e5126df8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_df.rename(columns={\"image\": \"Image_Index\", \"label\": \"Finding_Labels\"}, inplace=True)\n",
    "sampled_df.to_csv(\"./tum-ai-hackathon-model-implementation/full_data/Custom_Sampled_Dataset.csv\", sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79a108aa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_latest_p36",
   "language": "python",
   "name": "conda_pytorch_latest_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}