GC-LNM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# general\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.gridspec import GridSpec\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "import pickle\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "# local\n",
    "import preprocessor, univariate, evaluation\n",
    "\n",
    "from preprocessor import (\n",
    "    ENCODER,\n",
    "    onehot_encoder,\n",
    "    ordinal_encoder,\n",
    "    preprocessor_test\n",
    ")\n",
    "\n",
    "from univariate import (\n",
    "    get_univariate_table,\n",
    "    plot_univariate_auc\n",
    ")\n",
    "\n",
    "from evaluation import (\n",
    "    make_random_list,\n",
    "    evaluation_report,\n",
    "    plot_evaluation_report,\n",
    "    plot_calibration_curve_together,\n",
    "    plot_roc_curve_together,\n",
    "    plot_pr_curve_together,\n",
    "    plot_dca_together,\n",
    "    \n",
    "    plot_beeswarm_per_features,\n",
    "    plot_single_shap_waterfall\n",
    ")\n",
    "\n",
    "# SKLEARN\n",
    "\n",
    "from sklearn import set_config\n",
    "set_config(display=\"text\")  # displays text representation of estimators\n",
    "from sklearn.exceptions import FitFailedWarning\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer, IterativeImputer\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, KFold, ShuffleSplit\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "\n",
    "# Models\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "# Explain\n",
    "\n",
    "import shap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split\n",
    "\n",
    "data_train = pd.read_table('data/SDFEY_final.csv', sep=',', header=0)\n",
    "data_test = pd.read_table('data/SEER_final.csv', sep=',', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature name lists\n",
    "\n",
    "feature_names = ['Age', 'Sex', 'Location', 'Tumor size', 'T category, broad',\n",
    "                 'T category', 'Histology', 'SRCC', 'Grade']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train targets\n",
    "\n",
    "X_train = data_train[feature_names]\n",
    "X_test = data_test[feature_names]\n",
    "\n",
    "y_train = data_train['LNM']\n",
    "y_test = data_test['LNM']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_encode = ENCODER(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputer = IterativeImputer(estimator=BayesianRidge(),\n",
    "                           max_iter=1000,\n",
    "                           initial_strategy='mean',\n",
    "                           sample_posterior=True,\n",
    "                           random_state=2024) #\n",
    "\n",
    "X_train_impute = imputer.fit_transform(X_train_encode)\n",
    "X_train_impute = pd.DataFrame(X_train_impute, columns = X_train_encode.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = PowerTransformer()\n",
    "\n",
    "X_train_scale = scaler.fit_transform(X_train_impute)\n",
    "X_train_scale = pd.DataFrame(X_train_scale, columns=X_train_impute.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess external set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_scale = preprocessor_test(X_test, ENCODER, imputer, scaler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_final = X_train_scale\n",
    "X_test_final = X_test_scale"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Univariate FS using baseline models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_ = ShuffleSplit(n_splits = 100, test_size = 0.2, random_state = 1437)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator_list = [\n",
    "    LogisticRegression(), \n",
    "    SVC(),\n",
    "    KNeighborsClassifier(),\n",
    "    GaussianNB(),\n",
    "    DecisionTreeClassifier(),\n",
    "    HistGradientBoostingClassifier(),\n",
    "    RandomForestClassifier(),\n",
    "    MLPClassifier(),\n",
    "]\n",
    "\n",
    "name_list = ['LR', 'SVM', 'KNN', 'NB', 'DT', 'GBM', 'RF', 'ANN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table1, annot_table1 = get_univariate_table(estimator_list, \n",
    "                                            X_train_final, \n",
    "                                            name_list, \n",
    "                                            y_train, \n",
    "                                            cv = cv_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_univariate_auc(table1, annot_table1)\n",
    "fig.set_size_inches(15,7.5)\n",
    "fig.show()\n",
    "fig.savefig('Univariate FS.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## datasets after FS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_LR = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_SVM = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_KNN = X_train_final[['Tumor size', 'T category, broad', 'T category',\n",
    "                             'Histology_Intestinal type']]\n",
    "\n",
    "X_train_NB = X_train_final[['Age', \n",
    "                            'Tumor size', 'T category, broad', 'T category',\n",
    "                            'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_DT = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_GBM = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_RF = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_train_ANN = X_train_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_LR = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_SVM = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_KNN = X_test_final[['Tumor size', 'T category, broad', 'T category',\n",
    "                             'Histology_Intestinal type']]\n",
    "\n",
    "X_test_NB = X_test_final[['Age', \n",
    "                            'Tumor size', 'T category, broad', 'T category',\n",
    "                            'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_DT = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_GBM = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_RF = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                            'SRCC', 'Grade', \n",
    "                            'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                            'Histology_Diffuse type', 'Histology_Intestinal type']]\n",
    "\n",
    "X_test_ANN = X_test_final[['Tumor size', 'T category, broad', 'T category', \n",
    "                             'SRCC', 'Grade', \n",
    "                             'Location_Lower', 'Location_Middle', 'Location_Upper',\n",
    "                             'Histology_Diffuse type', 'Histology_Intestinal type']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = LogisticRegression().set_params(**{'penalty': 'l1', 'solver': 'liblinear', 'random_state': 15})\n",
    "SVM = SVC().set_params(**{'probability': True, 'C': 50, 'gamma': 0.0008, 'kernel': 'rbf', 'random_state': 15})\n",
    "KNN = KNeighborsClassifier().set_params(**{'algorithm': 'ball_tree', 'n_neighbors': 190, 'weights': 'uniform'})\n",
    "NB = GaussianNB().set_params(**{'var_smoothing': 0.01})\n",
    "DT = DecisionTreeClassifier().set_params(**{'criterion': 'gini', 'max_depth': 4, 'min_samples_leaf': 2, 'min_samples_split': 36, 'random_state': 15})\n",
    "GBM = HistGradientBoostingClassifier().set_params(**{'learning_rate': 0.1, 'max_iter': 60, 'min_samples_leaf': 105, 'random_state': 15})\n",
    "RF = RandomForestClassifier().set_params(**{'criterion': 'gini', 'max_depth': 8, 'n_estimators': 100, 'random_state': 15})\n",
    "ANN = MLPClassifier().set_params(**{'activation': 'relu', 'hidden_layer_sizes': (20, 50, 110), 'solver': 'sgd', 'random_state': 15})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR.fit(X_train_LR, y_train) \n",
    "SVM.fit(X_train_SVM, y_train)\n",
    "KNN.fit(X_train_KNN, y_train)\n",
    "NB.fit(X_train_NB, y_train)\n",
    "DT.fit(X_train_DT, y_train)\n",
    "GBM.fit(X_train_GBM, y_train)\n",
    "RF.fit(X_train_RF, y_train)\n",
    "ANN.fit(X_train_ANN, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator_list = [LR, SVM, KNN, NB, DT, GBM, RF, ANN]\n",
    "name_list = ['LR', 'SVM', 'KNN', 'NB', 'DT', 'GBM', 'RF', 'ANN']\n",
    "X_train_list = [X_train_LR, X_train_SVM, X_train_KNN, X_train_NB, X_train_DT, X_train_GBM, X_train_RF, X_train_ANN]\n",
    "X_test_list = [X_test_LR, X_test_SVM, X_test_KNN, X_test_NB, X_test_DT, X_test_GBM, X_test_RF, X_test_ANN]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_list = make_random_list(n_samples = 1000000, seed = 1437)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation IN TESTING SETS ↓"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics HM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table2, annot_table2 = evaluation_report(X_test_list, y_test, \n",
    "                                        estimator_list, name_list, \n",
    "                                        n_samples = 1000, \n",
    "                                        random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_evaluation_report(table2, annot_table2)\n",
    "fig.set_size_inches(15,5)\n",
    "fig.show()\n",
    "fig.savefig('Model Performance.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean1, legend_names1 = plot_roc_curve_together(X_test_list, y_test, \n",
    "                                                   estimator_list,\n",
    "                                                   name_list,\n",
    "                                                   n_samples = 1000,\n",
    "                                                   random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "plt.plot([0, 1], [0, 1], linestyle='dotted', color = 'black', label='Chance level (AUC = 0.5)')\n",
    "for i, legend_name in enumerate(legend_names1):\n",
    "    plots_mean1[i].plot(ax = ax,\n",
    "                       name = legend_name,\n",
    "                       markersize = 4,\n",
    "                       linestyle='-',\n",
    "                       linewidth = 2,\n",
    "                       alpha = 1\n",
    "                       )\n",
    "plt.title(f\"Mean ROC Curve\")\n",
    "plt.savefig('ROC curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PR curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean2, legend_names2 = plot_pr_curve_together(X_test_list, y_test, \n",
    "                                                  estimator_list,\n",
    "                                                  name_list,\n",
    "                                                  n_samples = 1000,\n",
    "                                                  random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "plt.axhline(y = 0.5, linestyle='dotted', color = 'black', label='Chance level (AUC = 0.5)')\n",
    "for i, legend_name in enumerate(legend_names2):\n",
    "    plots_mean2[i].plot(ax = ax,\n",
    "                       name = legend_name,\n",
    "                       markersize = 4,\n",
    "                       linestyle='-',\n",
    "                       linewidth = 2,\n",
    "                       alpha = 1\n",
    "                       )\n",
    "plt.title(f\"Mean PR Curve\")\n",
    "plt.savefig('PR curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize = (10, 10))\n",
    "\n",
    "ax = plot_dca_together(X_test_list, y_test, \n",
    "                        estimator_list,\n",
    "                        name_list,\n",
    "                        n_samples = 1000,\n",
    "                        random_list = random_list,\n",
    "                        ax = ax)\n",
    "\n",
    "fig.show()\n",
    "fig.savefig('Mean DCA.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calibration curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean, legend_names = plot_calibration_curve_together(X_test_list, y_test, \n",
    "                                                           estimator_list,\n",
    "                                                           name_list,\n",
    "                                                           dots_no = 21, \n",
    "                                                           n_samples = 1000,\n",
    "                                                           random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "for i, legend_name in enumerate(legend_names):\n",
    "    plots_mean[i].plot(ax = ax,\n",
    "                    name = legend_name,\n",
    "                    marker = 's', \n",
    "                    markersize = 4,\n",
    "                    linestyle = '-',\n",
    "                    linewidth = 2,\n",
    "                    alpha = 1\n",
    "                    )      \n",
    "\n",
    "plt.title(f\"Mean Calibration Curve\")\n",
    "plt.savefig('Calibration curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation IN TRAINING SETS ↓"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics HM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table3, annot_table3 = evaluation_report(X_train_list, y_train, \n",
    "                                        estimator_list, name_list, \n",
    "                                        n_samples = 1000, \n",
    "                                        random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_evaluation_report(table3, annot_table3)\n",
    "fig.set_size_inches(15,5)\n",
    "fig.show()\n",
    "fig.savefig('Model Performance.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean3, legend_names3 = plot_roc_curve_together(X_train_list, y_train, \n",
    "                                                   estimator_list,\n",
    "                                                   name_list,\n",
    "                                                   n_samples = 1000,\n",
    "                                                   random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "plt.plot([0, 1], [0, 1], linestyle='dotted', color = 'black', label='Chance level (AUC = 0.5)')\n",
    "for i, legend_name in enumerate(legend_names3):\n",
    "    plots_mean3[i].plot(ax = ax,\n",
    "                       name = legend_name,\n",
    "                       markersize = 4,\n",
    "                       linestyle='-',\n",
    "                       linewidth = 2,\n",
    "                       alpha = 1\n",
    "                       )\n",
    "plt.title(f\"Mean ROC Curve\")\n",
    "plt.savefig('ROC curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PR curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean4, legend_names4 = plot_pr_curve_together(X_train_list, y_train, \n",
    "                                                  estimator_list,\n",
    "                                                  name_list,\n",
    "                                                  n_samples = 1000,\n",
    "                                                  random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "plt.axhline(y = 0.5, linestyle='dotted', color = 'black', label='Chance level (AUC = 0.5)')\n",
    "for i, legend_name in enumerate(legend_names4):\n",
    "    plots_mean4[i].plot(ax = ax,\n",
    "                       name = legend_name,\n",
    "                       markersize = 4,\n",
    "                       linestyle='-',\n",
    "                       linewidth = 2,\n",
    "                       alpha = 1\n",
    "                       )\n",
    "plt.title(f\"Mean PR Curve\")\n",
    "plt.savefig('PR curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize = (10, 10))\n",
    "\n",
    "ax = plot_dca_together(X_train_list, y_train, \n",
    "                        estimator_list,\n",
    "                        name_list,\n",
    "                        n_samples = 1000,\n",
    "                        random_list = random_list,\n",
    "                        ax = ax)\n",
    "\n",
    "fig.show()\n",
    "fig.savefig('Mean DCA.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calibration curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_mean, legend_names = plot_calibration_curve_together(X_train_list, y_train, \n",
    "                                                           estimator_list,\n",
    "                                                           name_list,\n",
    "                                                           dots_no = 21, \n",
    "                                                           n_samples = 1000,\n",
    "                                                           random_list = random_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "gs = GridSpec(2, 2)\n",
    "ax = fig.add_subplot(gs[:2, :2])\n",
    "\n",
    "for i, legend_name in enumerate(legend_names):\n",
    "    plots_mean[i].plot(ax = ax,\n",
    "                    name = legend_name,\n",
    "                    marker = 's', \n",
    "                    markersize = 4,\n",
    "                    linestyle = '-',\n",
    "                    linewidth = 2,\n",
    "                    alpha = 1\n",
    "                    )      \n",
    "\n",
    "plt.title(f\"Mean Calibration Curve\")\n",
    "plt.savefig('Calibration curve.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# shap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = shap.Explainer(ANN.predict, X_test_ANN)\n",
    "explanation = explainer(X_test_ANN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.clf()\n",
    "shap.plots.beeswarm(explanation, \n",
    "                    plot_size = (15, 5), \n",
    "                    max_display=X_test_ANN.shape[1], \n",
    "                    show = False)\n",
    "plt.savefig('beeswarm.pdf', format='pdf', bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shap.plots.bar(explanation, clustering=False, max_display=18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature_name in ['T category', 'Tumor size', 'Histology_Intestinal type', \n",
    "                     'T category, broad', 'Location_Lower', 'Location_Upper', 'Location_Middle', \n",
    "                     'Histology_Diffuse type', 'Grade', 'SRCC']:\n",
    "    plot_beeswarm_per_features(explanation, name = feature_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explanation\n",
    "X_test = X_test\n",
    "model = ANN\n",
    "X_test_model = X_test_ANN\n",
    "id = 4\n",
    "\n",
    "plot_single_shap_waterfall(explanation, X_test, model, X_test_model, id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}