Build and train model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Skeleton Code\n",
    "\n",
    "The code below provides a skeleton for the model building & training component of your project. You can add/remove/build on code however you see fit, this is meant as a starting point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "from glob import glob\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import chain\n",
    "import sklearn.model_selection\n",
    "from random import sample \n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import EarlyStopping\n",
    "from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score, plot_precision_recall_curve, f1_score, confusion_matrix\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do some early processing of your metadata for easier model training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "## Below is some helper code to read all of your full image filepaths into a dataframe for easier manipulation\n",
    "\n",
    "all_xray_df = pd.read_csv('/data/Data_Entry_2017.csv')\n",
    "all_image_paths = {os.path.basename(x): x for x in \n",
    "                   glob(os.path.join('/data','images*', '*', '*.png'))}\n",
    "print('Scans found:', len(all_image_paths), ', Total Headers', all_xray_df.shape[0])\n",
    "all_xray_df['path'] = all_xray_df['Image Index'].map(all_image_paths.get)\n",
    "all_xray_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set unrealistic ages to NaN\n",
    "all_xray_df.replace(all_xray_df[all_xray_df['Patient Age']>100]['Patient Age'].values,np.nan, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Here you may want to create some extra columns in your table with binary indicators of certain diseases \n",
    "## rather than working directly with the 'Finding Labels' column\n",
    "\n",
    "# Todo\n",
    "def split_labels(df):\n",
    "    labels = np.unique(list(chain(*df['Finding Labels'].map(lambda x: x.split('|')).tolist())))\n",
    "    for i in labels:\n",
    "        df[i] = df['Finding Labels'].map(lambda y: 1.0 if i in y else 0)\n",
    "        \n",
    "split_labels(all_xray_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop the 'Unnamed: 11' column from the dataset\n",
    "all_xray_df.drop('Unnamed: 11', axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_xray_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_xray_df['Pneumonia Class'] = all_xray_df['Pneumonia'].map(lambda x: 'Positive' if x == 1 else 'Negative')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create your training and testing data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_splits(vargs):\n",
    "    \n",
    "#Initial split\n",
    "    train_data, val_data = sklearn.model_selection.train_test_split(vargs, test_size = 0.2, stratify = vargs['Pneumonia'])\n",
    "\n",
    "#want equal number of +ve/-ve pneumonia cases in train set\n",
    "    train_p_inds = train_data[train_data.Pneumonia==1].index.tolist()\n",
    "    train_np_inds = train_data[train_data.Pneumonia==0].index.tolist()\n",
    "\n",
    "    train_np_sample = sample(train_np_inds,len(train_p_inds))\n",
    "    train_data = train_data.loc[train_p_inds + train_np_sample]\n",
    "\n",
    "#want % of +ve pneumonia cases in validation set to be equal to natural occurence of the disease in the main dataset\n",
    "    val_p_inds = val_data[val_data.Pneumonia==1].index.tolist()\n",
    "    val_np_inds = val_data[val_data.Pneumonia==0].index.tolist()\n",
    "\n",
    "# The following code pulls a random sample of non-pneumonia data that's 4 times as big as the pneumonia sample\n",
    "    val_np_sample = sample(val_np_inds, 4*len(val_p_inds))\n",
    "    val_data = val_data.loc[val_p_inds + val_np_sample]\n",
    "    return  train_data, val_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating the training and validation sets\n",
    "train_data, val_data = create_splits(all_xray_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ANALYSE DISTRIBUTIONS IN TRAINING AND VALIDATION DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#defining age distribution fnc\n",
    "def age(df):\n",
    "    plt.hist(df['Patient Age'], bins = 10,)\n",
    "    plt.xlabel('age')\n",
    "    plt.ylabel('Number of People')\n",
    "    plt.title('Age Distribution in Dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#age distribution in training set\n",
    "age(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#majority of data is between 10 and 70 years of age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#age distribution in validation set\n",
    "age(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to plot gender demographics\n",
    "def gender(df):\n",
    "    df['Patient Gender'].value_counts().plot(kind='bar')\n",
    "    plt.xlabel('Gender')\n",
    "    plt.ylabel('Number of People')\n",
    "    plt.title('Gender Distribution in Dataset')\n",
    "    \n",
    "    return df['Patient Gender'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#invoking gender distribution fnc for train set\n",
    "train_gender_distribution = gender(train_data)\n",
    "train_gender_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#invoking gender distribution fnc for validation set\n",
    "val_gender_distribution = gender(val_data)\n",
    "val_gender_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def fnc to show position distribution\n",
    "def image_pos(df):\n",
    "    df['View Position'].value_counts().plot(kind='bar')\n",
    "    plt.xlabel('Image Position')\n",
    "    plt.ylabel('Number of People')\n",
    "    plt.title('Image Position Distribution')\n",
    "    return df['View Position'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#invoking position distribution fnc for training set\n",
    "train_image_position_distribution = image_pos(train_data)\n",
    "train_image_position_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#invoking position distribution fnc for validation set\n",
    "val_image_position_distribution = image_pos(val_data)\n",
    "val_image_position_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#determine the distribution of diseases comorbid with Pneumonia. Plotting the top 30 combinations\n",
    "train_data[train_data.Pneumonia == 1]['Finding Labels'].value_counts()[0:30].plot(kind = 'bar')\n",
    "plt.xlabel('Diseases Comorbid with Pneumonia')\n",
    "plt.ylabel('Number of People')\n",
    "plt.title('Distribution of Diseases comorbid with Pneumonia')\n",
    "train_disease_conjunction_pneumonia = train_data[train_data.Pneumonia == 1]['Finding Labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#determine the distribution of diseases comorbid with Pneumonia. Plotting the top 30 combinations\n",
    "val_data[val_data.Pneumonia == 1]['Finding Labels'].value_counts()[0:30].plot(kind = 'bar')\n",
    "plt.xlabel('Diseases Comorbid with Pneumonia')\n",
    "plt.ylabel('Number of People')\n",
    "plt.title('Distribution of Diseases comorbid with Pneumonia')\n",
    "val_disease_conjunction_pneumonia = val_data[val_data.Pneumonia == 1]['Finding Labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dropping the Pneumonia column since we have the Pneumonia Class column\n",
    "train_data.drop('Pneumonia', axis = 1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dropping the Pneumonia column since we have the Pneumonia Class column\n",
    "val_data.drop('Pneumonia', axis = 1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now we can begin our model-building & training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First suggestion: perform some image augmentation on your data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_image_augmentation():\n",
    "    \n",
    "    my_idg = ImageDataGenerator(rescale = 1. / 255.0, horizontal_flip = True, vertical_flip = False, height_shift_range = 0.1, width_shift_range = 0.1, rotation_range = 20,shear_range = 0.1, zoom_range = 0.1)\n",
    "    \n",
    "    \n",
    "    return my_idg\n",
    "\n",
    "#function to normalize images in validation dataset\n",
    "def my_image_val_normalize():\n",
    "    \n",
    "    my_idg = ImageDataGenerator(rescale = 1. / 255.0, horizontal_flip = False, vertical_flip = False)\n",
    "    \n",
    "    \n",
    "    return my_idg\n",
    "\n",
    "\n",
    "def make_train_gen(vargs):\n",
    "    \n",
    "    ## Create the actual generators using the output of my_image_augmentation for your training data\n",
    "    ## This generator uses a batch size of 32\n",
    "    idg = my_image_augmentation()\n",
    "    train_gen = idg.flow_from_dataframe(dataframe=vargs, directory=None, x_col = 'path',y_col = 'Pneumonia Class' ,class_mode = 'binary',target_size = (224,224), batch_size = 32)\n",
    "    \n",
    "\n",
    "    return train_gen\n",
    "\n",
    "def make_train_gen_2(vargs):\n",
    "    \n",
    "    ## Create the actual generators using the output of my_image_augmentation for your training data\n",
    "    ## This generator uses a batch size of 16\n",
    "    idg = my_image_augmentation()\n",
    "    train_gen = idg.flow_from_dataframe(dataframe=vargs, directory=None, x_col = 'path',y_col = 'Pneumonia Class' ,class_mode = 'binary',target_size = (224,224), batch_size = 16)\n",
    "    \n",
    "\n",
    "    return train_gen\n",
    "\n",
    "def make_train_gen_3(vargs):\n",
    "    \n",
    "    ## Create the actual generators using the output of my_image_augmentation for your training data\n",
    "    ## This generator uses a batch size of 64\n",
    "    idg = my_image_augmentation()\n",
    "    train_gen = idg.flow_from_dataframe(dataframe=vargs, directory=None, x_col = 'path',y_col = 'Pneumonia Class' ,class_mode = 'binary',target_size = (224,224), batch_size = 64)\n",
    "    \n",
    "\n",
    "    return train_gen\n",
    "\n",
    "# Generator for the validation dataset\n",
    "def make_val_gen(vargs):\n",
    "    \n",
    "    idg = my_image_val_normalize()\n",
    "    val_gen = idg.flow_from_dataframe(dataframe=vargs, directory=None, x_col = 'path',y_col = 'Pneumonia Class',class_mode = 'binary',target_size = (224,224), batch_size = 256)\n",
    "    \n",
    "    return val_gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the augmented training dataset with a batch size of 32\n",
    "train_gen = make_train_gen(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the augmented training dataset with a batch size of 16\n",
    "\n",
    "train_gen_2 = make_train_gen_2(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the augmented training dataset with a batch size of 64\n",
    "\n",
    "train_gen_3 = make_train_gen_3(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create normalized validation dataset\n",
    "val_gen = make_val_gen(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## May want to pull a single large batch of random validation data for testing after each epoch:\n",
    "valX, valY = val_gen.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## May want to look at some examples of our augmented training data. \n",
    "## This is helpful for understanding the extent to which data is being manipulated prior to training, \n",
    "## and can be compared with how the raw data look prior to augmentation\n",
    "\n",
    "t_x, t_y = next(train_gen)\n",
    "fig, m_axs = plt.subplots(4, 4, figsize = (16, 16))\n",
    "for (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n",
    "    c_ax.imshow(c_x[:,:,0], cmap = 'bone')\n",
    "    if c_y == 1: \n",
    "        c_ax.set_title('Pneumonia')\n",
    "    else:\n",
    "        c_ax.set_title('No Pneumonia')\n",
    "    c_ax.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build your model: \n",
    "\n",
    "Recommendation here to use a pre-trained network downloaded from Keras for fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loads the VGG 16 model and freezes all but the last CNN layer, returns the new model\n",
    "def load_pretrained_model_1():\n",
    "    \n",
    "    model = VGG16(include_top=True, weights='imagenet')\n",
    "    transfer_layer = model.get_layer('block5_pool')\n",
    "    vgg_model = Model(inputs = model.input, outputs = transfer_layer.output)\n",
    "    for layer in vgg_model.layers[0:17]:\n",
    "        layer.trainable = False\n",
    "    \n",
    "    \n",
    "    \n",
    "    return vgg_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Loads the VGG 16 model and freezes all but the last 2 CNN layer, returns the new model\n",
    "def load_pretrained_model_2():\n",
    "    \n",
    "    model = VGG16(include_top=True, weights='imagenet')\n",
    "    transfer_layer = model.get_layer('block5_pool')\n",
    "    vgg_model = Model(inputs = model.input, outputs = transfer_layer.output)\n",
    "    for layer in vgg_model.layers[0:16]:\n",
    "        layer.trainable = False\n",
    "    \n",
    "    \n",
    "    \n",
    "    return vgg_model\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model has 5 Dense layers and uses a dropout of 0.3 and is added on top of the VGG 16 model with all but its last CNN layer frozen\n",
    "\n",
    "def build_my_model_1(vgg_model):\n",
    "    \n",
    "\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.3))\n",
    "    my_model.add(Dense(4096, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.3))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.3))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.3))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model has 5 Dense layers and uses a dropout of 0.5 and is added on top of the VGG 16 model with all but its last CNN layer frozen\n",
    "\n",
    "\n",
    "def build_my_model_2(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(4096, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "    \n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    " \n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model has 4 Dense layers and uses a dropout of 0.5 and is added on top of the VGG 16 model with all but its last CNN layer frozen\n",
    "\n",
    "\n",
    "def build_my_model_3(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "  \n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##This model has 4 Dense layers and uses a dropout of 0.5 and is added on top of the VGG 16 model with all but its last 2 layers frozen\n",
    "\n",
    "def build_my_model_4(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    " \n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model uses a lr of 0.001\n",
    "def build_my_model_5(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    return my_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model uses a lr of 0.0005\n",
    "\n",
    "def build_my_model_6(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .0005)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model consists of just a single Dense layer\n",
    "def build_my_model_7(vgg_model):\n",
    "    \n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    \n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "        \n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model will eventually be trained on a training set with batch size = 16\n",
    "def build_my_model_8(vgg_model):\n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "    # my_model = Sequential()\n",
    "    # ....add your pre-trained model, and then whatever additional layers you think you might\n",
    "    # want for fine-tuning (Flatteen, Dense, Dropout, etc.)\n",
    "    \n",
    "    # if you want to compile your model within this function, consider which layers of your pre-trained model, \n",
    "    # you want to freeze before you compile \n",
    "    \n",
    "    # also make sure you set your optimizer, loss function, and metrics to monitor\n",
    "    \n",
    "    # Todo\n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model will eventually be trained on a training set with batch size = 64\n",
    "\n",
    "def build_my_model_9(vgg_model):\n",
    "    my_model = Sequential()\n",
    "    my_model.add(vgg_model)\n",
    "    my_model.add(Flatten())\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(1024, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(512, activation = 'relu'))\n",
    "    my_model.add(Dropout(0.5))\n",
    "    my_model.add(Dense(256, activation = 'relu'))\n",
    "    my_model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "\n",
    "    \n",
    "    optimizer = Adam(lr = .0001)\n",
    "    loss = 'binary_crossentropy'\n",
    "    metrics = ['binary_accuracy']\n",
    "    \n",
    "    my_model.compile(optimizer = optimizer, loss = loss, metrics = metrics)\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    return my_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_1=\"my_model_1-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_1 = ModelCheckpoint(weight_path_1, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_1 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_1 = [checkpoint_1, early_1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_2=\"my_model_2-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_2 = ModelCheckpoint(weight_path_2, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_2 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_2 = [checkpoint_2, early_2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_3=\"my_model_3-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_3 = ModelCheckpoint(weight_path_3, monitor= 'val_loss', verbose=1, save_best_only=True,mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_3 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_3 = [checkpoint_3, early_3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_4=\"my_model_4-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_4 = ModelCheckpoint(weight_path_4, monitor= 'val_loss', verbose=1,save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_4 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_4 = [checkpoint_4, early_4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_5=\"my_model_5-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_5 = ModelCheckpoint(weight_path_5, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_5 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_5 = [checkpoint_5, early_5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_6=\"my_model_6-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_6 = ModelCheckpoint(weight_path_6, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_6 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_6 = [checkpoint_6, early_6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_7=\"my_model_7-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_7 = ModelCheckpoint(weight_path_7, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_7 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_7 = [checkpoint_7, early_7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "weight_path_8=\"my_model_8-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_8 = ModelCheckpoint(weight_path_8, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_8 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_8 = [checkpoint_8, early_8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_path_9=\"my_model_9-{epoch:02d}-{val_loss:.2f}.hdf5\"\n",
    "\n",
    "checkpoint_9 = ModelCheckpoint(weight_path_9, monitor= 'val_loss', verbose=1, save_best_only=True, mode= 'min', save_weights_only = True)\n",
    "\n",
    "early_9 = EarlyStopping(monitor= 'val_loss', mode= 'min', patience=7)\n",
    "\n",
    "callbacks_list_9 = [checkpoint_9, early_9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start training! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading a pretrained Vgg 16 model with all but its last CNN layer frozen\n",
    "vgg_model_1 = load_pretrained_model_1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading a pretrained Vgg 16 model with all but its last 2 CNN layers frozen\n",
    "vgg_model_2 = load_pretrained_model_2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_1 = build_my_model_1(vgg_model_1)\n",
    "\n",
    "history_1 = my_model_1.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model stopped early as validation loss was not improving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_2 = build_my_model_2(vgg_model_1)\n",
    "\n",
    "history_2= my_model_2.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####Training stopped after 9 epchs as val loss did not improve\n",
    "#####BASED ON THE 2 MODELS ABOVE WE CONCLUDE THAT A DROPOUT OF 0.5 IS BETTER since both the loss and val loss is lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_3 = build_my_model_3(vgg_model_1)\n",
    "\n",
    "history_3 = my_model_3.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###MODEL 3 PERFORMS BETTER AND MORE STABLE. IT ALSO STOPPED AT 9 EPOCHS AS VAL LOSS WAS NOT IMPROVING\n",
    "###GO AHEAD WITH 4 DENSE LAYERS and dropout(0.5). We Use the 2nd epoch of model 3 as our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_4 = build_my_model_4(vgg_model_2)\n",
    "\n",
    "history_4 = my_model_4.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 4 has a higher loss than 3, we stick with 3. Go ahead with only last CNN frozen\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_5 = build_my_model_5(vgg_model_1)\n",
    "\n",
    "history_5 = my_model_5.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 3 still better since it has a lower loss and val loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_6 = build_my_model_6(vgg_model_1)\n",
    "\n",
    "history_6 = my_model_6.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 3 still better since it has a lower loss and val loss. Use lr = .0001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_7 = build_my_model_7(vgg_model_1)\n",
    "\n",
    "history_7 = my_model_7.fit_generator(train_gen, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 3 still better since it has a lower loss and val loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_8 = build_my_model_8(vgg_model_1)\n",
    "\n",
    "history_8 = my_model_8.fit_generator(train_gen_2, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 3 still better since it has a lower loss and val loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train your model\n",
    "\n",
    "# Todo\n",
    "\n",
    "my_model_9 = build_my_model_9(vgg_model_1)\n",
    "\n",
    "history_9 = my_model_9.fit_generator(train_gen_3, validation_data = (valX, valY),  epochs = 10,  callbacks = callbacks_list_9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Model 3 still better since it has a lower loss and val loss. Use batch size of 32\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### After training for some time, look at the performance of your model by plotting some performance statistics:\n",
    "\n",
    "Note, these figures will come in handy for your FDA documentation later in the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## After training, make some predictions to assess your model's overall performance\n",
    "## Note that detecting pneumonia is hard even for trained expert radiologists, \n",
    "## so there is no need to make the model perfect.\n",
    "my_model.load_weights(weight_path)\n",
    "pred_Y = new_model.predict(valX, batch_size = 32, verbose = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_auc(t_y, p_y):\n",
    "    \n",
    "    fig, c_ax = plt.subplots(1,1, figsize = (9, 9))\n",
    "    fpr, tpr, thresholds = roc_curve(t_y, p_y)\n",
    "    c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)'  % ('Pneumonia', auc(fpr, tpr)))\n",
    "    c_ax.legend()\n",
    "    c_ax.set_xlabel('False Positive Rate')\n",
    "    c_ax.set_ylabel('True Positive Rate')\n",
    "    \n",
    "    return fpr, tpr, thresholds\n",
    "    \n",
    "    \n",
    "def plot_precision_recall_curve(t_y, p_y):\n",
    "    fig, c_ax = plt.subplots(1,1, figsize = (9, 9))\n",
    "    precision, recall, thresholds = precision_recall_curve(t_y, p_y)\n",
    "    c_ax.plot(recall, precision, label = '%s (AP Score:%0.2f)'  % ('Pneumonia', average_precision_score(t_y,p_y)))\n",
    "    c_ax.legend()\n",
    "    c_ax.set_xlabel('Recall')\n",
    "    c_ax.set_ylabel('Precision')\n",
    "    \n",
    "    return precision, recall, thresholds\n",
    " \n",
    "def plot_f1_threshold(t_y, p_y):\n",
    "    fig, c_ax = plt.subplots(1,1, figsize = (9, 9))\n",
    "    precision, recall, thresholds = precision_recall_curve(t_y, p_y)\n",
    "    f1 = 2*(precision*recall)/(precision+recall)\n",
    "    c_ax.plot(f1[0:len(thresholds)], thresholds, label = 'F1 vs Thresholds')\n",
    "    c_ax.legend()\n",
    "    c_ax.set_xlabel('Threshold')\n",
    "    c_ax.set_ylabel('F1')\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "# function to calculate the F1 score\n",
    "def calc_f1(prec,recall):\n",
    "    \n",
    "    return (2*(prec*recall)/(prec+recall))\n",
    "    \n",
    "   \n",
    "def plot_history(history):\n",
    "    \n",
    "    plt.figure()\n",
    "    plt.plot(history.history[\"loss\"], label=\"train_loss\")\n",
    "    plt.plot(history.history[\"val_loss\"], label=\"val_loss\")\n",
    "    plt.plot(history.history[\"binary_accuracy\"], label=\"train_acc\")\n",
    "    plt.plot(history.history[\"val_binary_accuracy\"], label=\"val_acc\")\n",
    "    plt.title(\"Training Loss and Accuracy on Dataset\")\n",
    "    plt.xlabel(\"Epoch #\")\n",
    "    plt.ylabel(\"Loss/Accuracy\")\n",
    "    plt.legend(loc=\"lower left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_1)\n",
    "history_1.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_2)\n",
    "history_2.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_3)\n",
    "history_3.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_4)\n",
    "history_4.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_5)\n",
    "history_5.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_6)\n",
    "history_6.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_7)\n",
    "history_7.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_8)\n",
    "history_8.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## plot figures\n",
    "plot_history(history_9)\n",
    "history_9.history['loss']\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_3.save('my_model_3.hdf5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_3 = keras.models.load_model('my_model_3.hdf5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_20 = keras.models.load_model('my_model_3.hdf5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_20.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_model_3.load_weights(\"my_model_3-02-0.52.hdf5\")\n",
    "pred_Y_3 = my_model_3.predict(valX, batch_size = 32, verbose = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = plot_auc(valY, pred_Y_3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prec, recall, threshold_2 = plot_precision_recall_curve(valY, pred_Y_3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_f1_threshold(valY, pred_Y_3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BASED ON THE ABOVE GRAPH IT IS EVIDENT THAT A THRESHOLD of .41 CAN RESULT IN AN F1 Score over .38. We desire an F1 score over 0.38\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once you feel you are done training, you'll need to decide the proper classification threshold that optimizes your model's performance for a given metric (e.g. accuracy, F1, precision, etc.  You decide) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Find the threshold that optimize your model's performance,\n",
    "## and use that threshold to make binary classification. Make sure you take all your metrics into consideration.\n",
    "threshold_select = 0.41\n",
    "index = (np.abs(threshold_2 - threshold_select)).argmin()\n",
    "precision = prec[index]\n",
    "rec = recall[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_score = calc_f1(precision,rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('F1 Score is: {}'.format(f1_score))\n",
    "print('Precisions is: {}'.format(precision))\n",
    "print('Recall is: {}'.format(rec)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#A high recall helps us since it is beneficial for screening and this model will ideally be used for screening\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Let's look at some examples of true vs. predicted with our best model: \n",
    "\n",
    "# Todo\n",
    "\n",
    "fig, m_axs = plt.subplots(10, 10, figsize = (16, 16))\n",
    "i = 0\n",
    "for (c_x, c_y, c_ax) in zip(valX[0:100], valY[0:100], m_axs.flatten()):\n",
    "    c_ax.imshow(c_x[:,:,0], cmap = 'bone')\n",
    "    if c_y == 1: \n",
    "        if pred_Y_3[i] > .41:\n",
    "            c_ax.set_title('1, 1')\n",
    "        else:\n",
    "            c_ax.set_title('1, 0')\n",
    "    else:\n",
    "        if pred_Y_3[i] > .41: \n",
    "            c_ax.set_title('0, 1')\n",
    "        else:\n",
    "            c_ax.set_title('0, 0')\n",
    "    c_ax.axis('off')\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Just save model architecture to a .json:\n",
    "\n",
    "model_json = my_model_3.to_json()\n",
    "with open(\"my_model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}