code_08_randomforest_using_flowsom_result.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation, Concatenate\n",
    "from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, Input\n",
    "from keras import backend as K\n",
    "from keras.models import load_model\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np\n",
    "import random\n",
    "from keras.utils.vis_utils import plot_model\n",
    "import keras.callbacks\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import TSNE\n",
    "import seaborn as sns\n",
    "from numpy.random import seed; seed(123)\n",
    "from tensorflow import set_random_seed; set_random_seed(123)\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from statsmodels.formula.api import ols\n",
    "from statsmodels.stats.anova import anova_lm\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### test flowSOM  #####\n",
    "processed_data = pickle.load( open( \"result_04_processed_data_no_scale.obj\", \"rb\" ) )\n",
    "cytof_files = processed_data[\"cytof_files\"]\n",
    "expr_list = processed_data[\"expr_list\"]\n",
    "\n",
    "r1 = [t1==t1 for t1 in cytof_files.CMV_Ab] \n",
    "cytof_files = cytof_files.loc[r1,:]\n",
    "expr_list = expr_list[r1]\n",
    "print(expr_list.shape)\n",
    "\n",
    "gating_result = pd.read_csv(\"result_07_summary_statistics.csv\")\n",
    "\n",
    "dat = pd.merge(cytof_files, gating_result, how='inner', left_on=\"name\",right_on=[\"fcs_files\"])\n",
    "coln = gating_result.columns[1:]\n",
    "y = dat.CMV_Ab.values> 2\n",
    "x = dat[coln].values\n",
    "x = np.nan_to_num(x)\n",
    "print(x.shape);print(y.shape)\n",
    "\n",
    "##### split train, validation and test######\n",
    "train_id = [i for i in range(len(x)) if dat.study_accession.iloc[i] not in [\"SDY515\",\"SDY519\"]]\n",
    "valid_id = [i for i in range(len(x)) if dat.study_accession.iloc[i]==\"SDY515\"]\n",
    "test_id = [i for i in range(len(x)) if dat.study_accession.iloc[i]==\"SDY519\"]\n",
    "\n",
    "x_train = x[train_id]\n",
    "x_valid = x[valid_id]\n",
    "x_test = x[test_id]\n",
    "\n",
    "y_train = y[train_id]\n",
    "y_valid = y[valid_id]\n",
    "y_test = y[test_id]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=1000)\n",
    "clf.fit(x_train, y_train)\n",
    "\n",
    "\n",
    "y_true = y_test\n",
    "y_scores = clf.predict_proba(x_test)[:,1]\n",
    "#print(y_scores)\n",
    "print(roc_auc_score(y_true, y_scores))\n",
    "\n",
    "with open(\"result_08_flowSOM_RF_ROC.obj\", \"wb\") as f:\n",
    "    pickle.dump({\"true\":y_test,\"score\":y_scores}, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### load data #####\n",
    "processed_data = pickle.load( open( \"result_04_processed_data_no_scale.obj\", \"rb\" ) )\n",
    "cytof_files = processed_data[\"cytof_files\"]\n",
    "expr_list = processed_data[\"expr_list\"]\n",
    "\n",
    "r1 = [t1==t1 for t1 in cytof_files.CMV_Ab] \n",
    "cytof_files = cytof_files.loc[r1,:]\n",
    "expr_list = expr_list[r1]\n",
    "print(expr_list.shape)\n",
    "\n",
    "gating_result = pd.read_csv(\"result_06_summary_statistics.csv\")\n",
    "\n",
    "dat = pd.merge(cytof_files, gating_result, how='inner', left_on=\"name\",right_on=[\"fcs_files\"])\n",
    "coln = gating_result.columns[1:]\n",
    "y = dat.CMV_Ab.values> 2\n",
    "x = dat[coln].values\n",
    "x = np.nan_to_num(x)\n",
    "print(x.shape);print(y.shape)\n",
    "\n",
    "##### split train, validation and test######\n",
    "train_id = [i for i in range(len(x)) if dat.study_accession.iloc[i] not in [\"SDY515\",\"SDY519\"]]\n",
    "valid_id = [i for i in range(len(x)) if dat.study_accession.iloc[i]==\"SDY515\"]\n",
    "test_id = [i for i in range(len(x)) if dat.study_accession.iloc[i]==\"SDY519\"]\n",
    "\n",
    "x_train = x[train_id]\n",
    "x_valid = x[valid_id]\n",
    "x_test = x[test_id]\n",
    "\n",
    "y_train = y[train_id]\n",
    "y_valid = y[valid_id]\n",
    "y_test = y[test_id]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=1000)\n",
    "clf.fit(x_train, y_train)\n",
    "\n",
    "\n",
    "y_true = y_test\n",
    "y_scores = clf.predict_proba(x_test)[:,1]\n",
    "#print(y_scores)\n",
    "print(roc_auc_score(y_true, y_scores))\n",
    "\n",
    "with open(\"result_08_10k_RF_ROC.obj\", \"wb\") as f:\n",
    "    pickle.dump({\"true\":y_test,\"score\":y_scores}, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "CyTOF_DL",
   "language": "python",
   "name": "cytof_dl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}