SeedCoatColour_ReducedInput_MCC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMPORT STATEMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: pandas in ./.local/lib/python3.6/site-packages (1.1.5)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (1.17.3)\n",
      "Requirement already satisfied: sklearn in ./.local/lib/python3.6/site-packages (0.0)\n",
      "Requirement already satisfied: xgboost in ./.local/lib/python3.6/site-packages (1.3.1)\n",
      "Requirement already satisfied: matplotlib in ./.local/lib/python3.6/site-packages (3.3.3)\n",
      "Requirement already satisfied: tensorflow==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0+nv)\n",
      "Requirement already satisfied: keras==2.2.4 in ./.local/lib/python3.6/site-packages (2.2.4)\n",
      "Requirement already satisfied: fastai in ./.local/lib/python3.6/site-packages (2.2.2)\n",
      "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (2.8.1)\n",
      "Requirement already satisfied: scikit-optimize in ./.local/lib/python3.6/site-packages (0.8.1)\n",
      "Requirement already satisfied: scikit-learn==0.21 in ./.local/lib/python3.6/site-packages (0.21.0)\n",
      "Requirement already satisfied: graphviz in ./.local/lib/python3.6/site-packages (0.16)\n",
      "Requirement already satisfied: pytz>=2017.2 in ./.local/lib/python3.6/site-packages (from pandas) (2020.5)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.4.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in ./.local/lib/python3.6/site-packages (from matplotlib) (8.1.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in ./.local/lib/python3.6/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in ./.local/lib/python3.6/site-packages (from matplotlib) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in ./.local/lib/python3.6/site-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (0.8.1)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (0.2.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.14.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.1.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.1.0)\n",
      "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (0.2.2)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (0.34.2)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (0.9.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (3.2.0)\n",
      "Requirement already satisfied: tensorboard<2.2.0,>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (2.1.1)\n",
      "Requirement already satisfied: tensorflow-estimator<2.2.0,>=2.1.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (2.1.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.0.8)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.12.1)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (1.27.2)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.1.0) (3.11.3)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (2.9.0)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (5.3.1)\n",
      "Requirement already satisfied: fastcore<1.4,>=1.3.8 in ./.local/lib/python3.6/site-packages (from fastai) (1.3.18)\n",
      "Requirement already satisfied: packaging in ./.local/lib/python3.6/site-packages (from fastai) (20.8)\n",
      "Requirement already satisfied: spacy in ./.local/lib/python3.6/site-packages (from fastai) (2.3.5)\n",
      "Requirement already satisfied: torchvision<0.9,>=0.8 in ./.local/lib/python3.6/site-packages (from fastai) (0.8.2)\n",
      "Requirement already satisfied: pip in /usr/local/lib/python3.6/dist-packages (from fastai) (20.0.2)\n",
      "Requirement already satisfied: torch<1.8,>=1.7.0 in ./.local/lib/python3.6/site-packages (from fastai) (1.7.1)\n",
      "Requirement already satisfied: fastprogress>=0.2.4 in ./.local/lib/python3.6/site-packages (from fastai) (1.0.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from fastai) (2.23.0)\n",
      "Requirement already satisfied: pyaml>=16.9 in ./.local/lib/python3.6/site-packages (from scikit-optimize) (20.4.0)\n",
      "Requirement already satisfied: joblib>=0.11 in ./.local/lib/python3.6/site-packages (from scikit-optimize) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (3.2.1)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.11.3)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.0.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.4.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (46.0.0)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (0.7.4)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (3.0.5)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai) (4.43.0)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (0.8.0)\n",
      "Requirement already satisfied: thinc<7.5.0,>=7.4.1 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (7.4.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (1.0.5)\n",
      "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (1.0.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (2.0.5)\n",
      "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (1.0.5)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in ./.local/lib/python3.6/site-packages (from spacy->fastai) (1.1.3)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in ./.local/lib/python3.6/site-packages (from torch<1.8,>=1.7.0->fastai) (0.8)\n",
      "Requirement already satisfied: typing-extensions in ./.local/lib/python3.6/site-packages (from torch<1.8,>=1.7.0->fastai) (3.7.4.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (2.9)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (1.25.8)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->fastai) (2019.11.28)\n",
      "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (4.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (4.0.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.2.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.3.0)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy->fastai) (1.5.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (3.1.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->fastai) (3.1.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.3.3 is available.\r\n",
      "You should consider upgrading via the '/usr/bin/python -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!TMPDIR=/home/mgill/ pip install --cache-dir=/home/mgill/ --build /home/mgill/ pandas numpy sklearn xgboost matplotlib tensorflow==2.1.0 keras==2.2.4 fastai python-dateutil scikit-optimize scikit-learn==0.21 graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n",
      "2.2.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.impute import SimpleImputer\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import tensorflow as tf; print(tf.__version__)\n",
    "import keras; print(keras.__version__)\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from keras.regularizers import *\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from fastai.tabular.all import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, auc\n",
    "from random import randint\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.preprocessing import sequence\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.layers import Activation\n",
    "from math import sqrt\n",
    "from statistics import mean\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "import skopt\n",
    "from skopt.searchcv import BayesSearchCV\n",
    "from skopt.space import Real, Categorical, Integer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import pickle\n",
    "from collections import OrderedDict\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from scipy import interp\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_prep_data(tt_file, ho_file):\n",
    "    imp = SimpleImputer(missing_values='./.', strategy='most_frequent')\n",
    "    my_list = []\n",
    "    x = 0 \n",
    "    for chunk in pd.read_csv(tt_file, chunksize=10000, index_col=\"Unnamed: 0\"):\n",
    "        x=x+10000\n",
    "        chunk = chunk.T\n",
    "        if 'Value' in chunk.columns:\n",
    "            #does the selecting of pheno array for application ML\n",
    "            chunk[\"Value\"] = pd.to_numeric(chunk[\"Value\"], downcast=\"float\")\n",
    "            tt_pheno = chunk[\"Value\"].to_numpy()\n",
    "            #reshapes it so its not a 1D array\n",
    "            print(tt_pheno.shape)\n",
    "            tt_pheno = np.reshape(tt_pheno,(len(tt_pheno),1))\n",
    "            print(tt_pheno.shape)\n",
    "            chunk = chunk.drop(columns=['Value'])\n",
    "        headers = chunk.columns\n",
    "        row_idx = chunk.index\n",
    "        chunk = imp.fit_transform(chunk) #SHOULD TURN ./. into the most common for each column\n",
    "        #since imputing makes a numpy array have to turn back into PD for label encoding\n",
    "        chunk = pd.DataFrame(data = chunk, index = row_idx, columns = headers)\n",
    "        my_list.append(chunk)\n",
    "        print(x)\n",
    "    tt_vcf = pd.concat(my_list, axis = 1)\n",
    "    my_list = []\n",
    "    x=0\n",
    "    for chunk in pd.read_csv(ho_file, chunksize=10000, index_col=\"Unnamed: 0\"):\n",
    "        x=x+10000\n",
    "        chunk = chunk.T\n",
    "        if 'Value' in chunk.columns:\n",
    "            #does the selecting of pheno array for application ML\n",
    "            chunk[\"Value\"] = pd.to_numeric(chunk[\"Value\"], downcast=\"float\")\n",
    "            ho_pheno = chunk[\"Value\"].to_numpy()\n",
    "            #reshapes it so its not a 1D array\n",
    "            print(ho_pheno.shape)\n",
    "            ho_pheno = np.reshape(ho_pheno,(len(ho_pheno),1))\n",
    "            print(ho_pheno.shape)\n",
    "            chunk = chunk.drop(columns=['Value'])\n",
    "        headers = chunk.columns\n",
    "        row_idx = chunk.index\n",
    "        chunk = imp.fit_transform(chunk) #SHOULD TURN ./. into the most common for each column\n",
    "        #since imputing makes a numpy array have to turn back into PD for label encoding\n",
    "        chunk = pd.DataFrame(data = chunk, index = row_idx, columns = headers)\n",
    "        my_list.append(chunk)\n",
    "        print(x)\n",
    "    ho_vcf = pd.concat(my_list, axis = 1)\n",
    "    print(tt_vcf.shape)\n",
    "    print(ho_vcf.shape)\n",
    "    return tt_vcf, ho_vcf, tt_pheno, ho_pheno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt_vcf, ho_vcf, tt_pheno, ho_pheno = new_prep_data(\"SCC_Merged_filtered.csv_train_testQTL_SNPS.csv\", \"SCC_Merged_filtered.csv_holdoutQTL_SNPS.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if it hasn't been run and saved before\n",
    "ohe = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
    "ohe = ohe.fit(tt_vcf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(ohe, open(\"SCC_QTL_ohe.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if need or have new holdout data etc.\n",
    "ohe = pickle.load(open(\"SCC_QTL_ohe.dat\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tt_vcf.shape)\n",
    "tt_vcf = ohe.transform(tt_vcf)\n",
    "print(tt_vcf.shape)\n",
    "print(ho_vcf.shape)\n",
    "ho_vcf = ohe.transform(ho_vcf)\n",
    "print(ho_vcf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_snp_from_header(ohe,snp_num):\n",
    "    count = 0\n",
    "    snp = \"Not found\"\n",
    "    found = False\n",
    "    i = 0\n",
    "    while i < len(ohe.categories_) and (found == False):\n",
    "        j = 0\n",
    "        while j < len(ohe.categories_[i]):\n",
    "            if(count == snp_num):\n",
    "                snp = ohe.categories_[i][j]\n",
    "                found = True\n",
    "                break\n",
    "            count = count + 1\n",
    "            j = j + 1\n",
    "        i = i + 1\n",
    "    return snp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TESTING IF IT WORKS\n",
    "my_snp = find_snp_from_header(ohe, 462793)\n",
    "print(my_snp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBOOST ML Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tt_vcf.shape)\n",
    "print(tt_pheno.shape)\n",
    "seed = randint(0,5000)\n",
    "X_train, X_test, y_train, y_test = train_test_split(tt_vcf, tt_pheno, test_size=0.2, random_state=seed)\n",
    "print(X_test.shape)\n",
    "print(\"seed is \" + str(seed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "space ={'learning_rate': Real(0.01, 1.0, 'log-uniform'),\n",
    "        'min_child_weight': Integer(0, 10),\n",
    "        'max_depth': Integer(0, 50),\n",
    "        'max_delta_step': Integer(0, 20),\n",
    "        'subsample': Real(0.01, 1.0, 'uniform'),\n",
    "        'colsample_bytree': Real(0.01, 1.0, 'uniform'),\n",
    "        'colsample_bylevel': Real(0.01, 1.0, 'uniform'),\n",
    "        'reg_lambda': Real(1e-9, 1000, 'log-uniform'),\n",
    "        'reg_alpha': Real(1e-9, 1.0, 'log-uniform'),\n",
    "        'gamma': Real(1e-9, 0.5, 'log-uniform'),\n",
    "        'min_child_weight': Integer(0, 5),\n",
    "        'n_estimators': Integer(50, 200),\n",
    "        'scale_pos_weight': Real(1e-6, 500, 'log-uniform')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_step(optim_result):\n",
    "    \"\"\"\n",
    "    Callback meant to view scores after\n",
    "    each iteration while performing Bayesian\n",
    "    Optimization in Skopt\"\"\"\n",
    "    score = xgb_bayes_search.best_score_\n",
    "    print(\"best score: %s\" % score)\n",
    "    if score >= 0.98:\n",
    "        print('Interrupting!')\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbcl = xgb.XGBClassifier(objective='multi:softmax', num_class=4)\n",
    "xgb_bayes_search = BayesSearchCV(xgbcl, space, n_iter=32, # specify how many iterations\n",
    "                                    scoring=None, n_jobs=1, cv=5, verbose=3, random_state=42, n_points=12,\n",
    "                                 refit=True)\n",
    "xgb_bayes_search.fit(X_train, y_train.ravel(), callback = on_step)\n",
    "print(xgb_bayes_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(xgb_bayes_search.best_params_)\n",
    "model = xgb.XGBClassifier(**xgb_bayes_search.best_params_, objective='multi:softmax', num_class=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##if not in same run as optimisation\n",
    "best_params = OrderedDict([('colsample_bylevel', 0.25617325301227906), ('colsample_bytree', 0.7083937150495909), ('gamma', 2.41812432168581e-07), ('learning_rate', 0.13965555720269418), ('max_delta_step', 10), ('max_depth', 27), ('min_child_weight', 1), ('n_estimators', 76), ('reg_alpha', 3.178148842971562e-08), ('reg_lambda', 0.005381781269387993), ('scale_pos_weight', 0.23835043249575294), ('subsample', 0.9559763235078597)])\n",
    "model = xgb.XGBClassifier(**best_params, objective='multi:softmax', num_class=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## DO MORE STUFF HERE when OPTIMISED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_k_fold(m, x, y, k, hx, hy):\n",
    "    #model: xgboost model, should be with the best params available\n",
    "    #x: input data (eg. all samples and SNPS)\n",
    "    #y: labels\n",
    "    #k: number of folds for cross validation\n",
    "    cv = StratifiedKFold(n_splits=k,shuffle=False)\n",
    "    fig1 = plt.figure(figsize=[12,12])\n",
    "\n",
    "    tprs = []\n",
    "    aucs = []\n",
    "    results = []\n",
    "    mean_fpr = np.linspace(0,1,100)\n",
    "    high = 0\n",
    "    best = m\n",
    "    i = 1\n",
    "    for train,test in cv.split(x,y):\n",
    "        prediction = m.fit(x[train],y[train].ravel()).predict_proba(x[test])\n",
    "        print(\"variables for auroc curve done. Processing fold accuracy + checking best model\")\n",
    "        y_pred = m.predict(x[test])\n",
    "        predictions = [round(value) for value in y_pred]\n",
    "        #sees how accurate the model was when testing the test set\n",
    "        accuracy = accuracy_score(y[test], predictions)\n",
    "        pcent = accuracy * 100.0\n",
    "        print(\"The accuracy of this model is\" + str(pcent))\n",
    "        if(pcent > high):\n",
    "            high = pcent\n",
    "            best = m\n",
    "       # fpr, tpr, t = roc_curve(y[test], prediction[:, 1])\n",
    "       # tprs.append(interp(mean_fpr, fpr, tpr))\n",
    "       # roc_auc = auc(fpr, tpr)\n",
    "       # aucs.append(roc_auc)\n",
    "        results.append(pcent)\n",
    "       # plt.plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))\n",
    "        i= i+1\n",
    "\n",
    "   # plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'black')\n",
    "   # mean_tpr = np.mean(tprs, axis=0)\n",
    "   # mean_auc = auc(mean_fpr, mean_tpr)\n",
    "   # plt.plot(mean_fpr, mean_tpr, color='blue',\n",
    "    #         label=r'Mean ROC (AUC = %0.2f )' % (mean_auc),lw=2, alpha=1)\n",
    "    \n",
    "    holdout_pred = best.predict(hx)\n",
    "    predictions = [round(value) for value in holdout_pred]\n",
    "    #sees how accurate the model was when testing the test set\n",
    "    accuracy = accuracy_score(hy, predictions)\n",
    "    pcent = accuracy * 100.0\n",
    "    print(pcent)\n",
    "    xgb_predictions = best.predict(hx)\n",
    "    xgb_probs = best.predict_proba(hx)[:, 1]\n",
    "    #model_fpr, model_tpr, my_roccy = evaluate_model(xgb_predictions, xgb_probs, hy)\n",
    "    #plt.plot(model_fpr, model_tpr, 'r', label = 'Holdout Data'+my_roccy, lw=2)\n",
    "    \n",
    "    #plt.xlabel('False Positive Rate')\n",
    "    #plt.ylabel('True Positive Rate')\n",
    "    #plt.title('ROC Flower Colour Training Model & Holdout Data')\n",
    "    #plt.legend(loc=\"lower right\")\n",
    "    #plt.show()\n",
    "\n",
    "    print(\"Training Testing Accuracy: %.2f%% (%.2f%%)\" % (np.mean(results), np.std(results)))\n",
    "    print(\"Holdout Accuracy: %.2f%%\" % (pcent))\n",
    "    \n",
    "    return best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tt_vcf.shape)\n",
    "print(tt_pheno.shape)\n",
    "print(ho_vcf.shape)\n",
    "print(ho_pheno.shape)\n",
    " #if optimised in same session, other enter manually below\n",
    "#this function should average out 10 folds and training, with inital params optimised\n",
    "#average accuracy and std should be calculated along with a nice AUROC graph of train/test models\n",
    "#best model should be extracted for use on holdout set\n",
    "best_model = eval_k_fold(model, tt_vcf, tt_pheno, 10, ho_vcf, ho_pheno)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(best_model, open(\"SCC_kfold_10_XGB_QTL.pickle.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only load if not generated in same session\n",
    "best_model = pickle.load(open(\"SCC_kfold_10_XGB_QTL.pickle.dat\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## DO SNPS of importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import plot_importance\n",
    "from matplotlib import pyplot\n",
    "#best_model = pickle.load(open(\"FC_kfold_10_tt_from_all.pickle.dat\", \"rb\"))\n",
    "plt.figure(figsize = (20, 20))\n",
    "plot_importance(best_model, max_num_features=15, importance_type='gain', height=0.3)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function essentially returns an array of dataframe headers the length of OHE'd input SNPs for training data\n",
    "#EG. It will be able to determine that feature 357310 is Gm13_17683957 but not what allele it is\n",
    "#eg. feature 357309 357310 and 357311 may all be one hot encoded versions of all possible values of Gm13_17683957\n",
    "#iterating through the saved OHE will by able to determine what specific allele the feature is but cannot determine\n",
    "#what SNP header it belongs to. Therefore combining these two methods you can determine both allele and SNP\n",
    "snp = []\n",
    "imp = SimpleImputer(missing_values='./.', strategy='most_frequent')\n",
    "fs_ohe = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
    "x = 0\n",
    "n_headers = []\n",
    "le = LabelEncoder()\n",
    "#while (i < 10):\n",
    "for chunk in pd.read_csv(\"SCC_Merged_filtered.csv_train_testQTL_SNPS.csv\", chunksize=10000, index_col=\"Unnamed: 0\"):\n",
    "    chunk = chunk.T\n",
    "    if 'Value' in chunk.columns:\n",
    "        print(\"dropping value so it doesn't include that in headers\")\n",
    "        chunk = chunk.drop(columns=['Value'])\n",
    "    headers = chunk.columns\n",
    "    row_idx = chunk.index\n",
    "    chunk = imp.fit_transform(chunk) #SHOULD TURN ./. into the most common for each column\n",
    "    #since imputing makes a numpy array have to turn back into PD for label encoding\n",
    "    chunk = pd.DataFrame(data = chunk, index = row_idx, columns = headers)\n",
    "    chunk = chunk.apply(lambda col: le.fit_transform(col))\n",
    "    c_headers = chunk.columns\n",
    "    y = 0\n",
    "    for column in chunk:\n",
    "        d = (chunk[column].nunique())\n",
    "        n_headers.extend([c_headers[y] for i in range(d)])\n",
    "        #print(n_headers)\n",
    "        #print(l)\n",
    "        #n_headers.append(c_headers[y] * d)\n",
    "        #print(n_headers)\n",
    "        y = y + 1\n",
    "    #to double check that it would indeed be one hot encoded with this amount of columns\n",
    "    chunk = fs_ohe.fit_transform(chunk)\n",
    "    x = x + chunk.shape[1]\n",
    "    print(\"my X value is: \" + str(x))\n",
    "    print(chunk.shape)\n",
    "    print(\"my header list is: \" + str(len(n_headers)))\n",
    "print(len(n_headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "figure(num=None, figsize=(10, 10), dpi=100, facecolor='w', edgecolor='k')\n",
    "fs = [181106,56313,581117,214419,133458,214671,214582,214543,213998,214557,214545,214000,331158,30222,214531]\n",
    "scores = [125,92.7,57.8,34.9,27.5,22.4,20.3,20.2,19.9,16.9,16.8,16.1,15.51,15.5,13.1]\n",
    "snp_label = []\n",
    "for jj in fs:\n",
    "    jj_allele = find_snp_from_header(ohe, jj)\n",
    "    this_snp = (n_headers[jj] + ' ('+str(jj_allele)+')')\n",
    "    print(this_snp)\n",
    "    snp_label.append(this_snp)\n",
    "snp_label.reverse()\n",
    "scores.reverse()\n",
    "print(len(scores))\n",
    "print(len(snp_label))\n",
    "plt.barh(snp_label,scores)\n",
    "plt.title('SNP Importance XGBoost Seed Coat Colour')\n",
    "plt.ylabel('SNP Label')\n",
    "plt.xlabel('Relative F_Score (GAIN)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_dict = best_model.get_booster().get_score(importance_type=\"gain\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_f_header(fn,n_headers,ohe):\n",
    "    fn = fn[1:]\n",
    "    fn = int(fn)\n",
    "    allele = find_snp_from_header(ohe, fn)\n",
    "    this_snp = (n_headers[fn] + ' ('+str(allele)+')')\n",
    "    return this_snp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert feature to actual SNP name\n",
    "i = 0\n",
    "new_dict = {}\n",
    "for key in my_dict:\n",
    "    new_key = rename_f_header(key, n_headers, ohe)\n",
    "    new_dict[new_key] = my_dict[key]\n",
    "    i = i + 1\n",
    "    print(str(i))\n",
    "    if(my_dict):\n",
    "        continue\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_fi = pd.Series(new_dict)\n",
    "print(new_fi)\n",
    "df = new_fi.to_frame()\n",
    "df = df.rename(columns = {0:'F_Score(GAIN)'})\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "figure(num=None, figsize=(10, 10), dpi=100, facecolor='w', edgecolor='k')\n",
    "indexes = df.nlargest(20, \"F_Score(GAIN)\").index\n",
    "values = df.nlargest(20, \"F_Score(GAIN)\").values.ravel()\n",
    "indexes = indexes[::-1]\n",
    "values = values[::-1]\n",
    "plt.barh(indexes, values)\n",
    "plt.title('SNP Importance XGBoost Seed Coat Colour')\n",
    "plt.ylabel('SNP Label')\n",
    "plt.xlabel('Relative F_Score (GAIN)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate figure object\n",
    "figure(num=None, figsize=(10, 10), dpi=100, facecolor='w', edgecolor='k')\n",
    "#load in the 20 lardest values and their SNP label\n",
    "indexes = df.nlargest(20, \"F_Score(GAIN)\").index\n",
    "values = df.nlargest(20, \"F_Score(GAIN)\").values.ravel()\n",
    "#reverse to make the largest be at the front\n",
    "indexes = indexes[::-1]\n",
    "values = values[::-1]\n",
    "#for each different chromosome you want to colour add a index(*_i) and value (*_v) array\n",
    "#black would be colour for singular/notinteresting chromosomes\n",
    "r_i = []\n",
    "r_v = []\n",
    "b_i = []\n",
    "b_v = []\n",
    "g_i = []\n",
    "g_v = []\n",
    "y_i = []\n",
    "y_v = []\n",
    "bl_i = []\n",
    "bl_v = []\n",
    "p_i = []\n",
    "p_v = []\n",
    "br_i = []\n",
    "br_v = []\n",
    "pu_i = []\n",
    "pu_v = []\n",
    "#for each value in the top n (default 20) check which chromosome it belongs to and add it to the colour array\n",
    "i = 0\n",
    "while i < len(indexes):\n",
    "    if('Gm12' in indexes[i]):\n",
    "        r_i.append(indexes[i])\n",
    "        r_v.append(values[i])\n",
    "    elif('Gm08' in indexes[i]):\n",
    "        b_i.append(indexes[i])\n",
    "        b_v.append(values[i])\n",
    "    elif('Gm06' in indexes[i]):\n",
    "        g_i.append(indexes[i])\n",
    "        g_v.append(values[i])\n",
    "    elif('Gm02' in indexes[i]):\n",
    "        y_i.append(indexes[i])\n",
    "        y_v.append(values[i])\n",
    "    elif('Gm19' in indexes[i]):\n",
    "        p_i.append(indexes[i])\n",
    "        p_v.append(values[i])\n",
    "   # elif('Gm04' in indexes[i]):\n",
    "   #     br_i.append(indexes[i])\n",
    "   #     br_v.append(values[i])\n",
    "   # elif('Gm13' in indexes[i]):\n",
    "   #     pu_i.append(indexes[i])\n",
    "   #     pu_v.append(values[i])\n",
    "    else:\n",
    "        bl_i.append(indexes[i])\n",
    "        bl_v.append(values[i])\n",
    "    i = i + 1\n",
    "#plot each of the arrays with appropriate colour and label graph\n",
    "plt.barh(bl_i, bl_v, color=\"black\")\n",
    "plt.barh(br_i, br_v, color=\"brown\")\n",
    "plt.barh(pu_i, pu_v, color=\"purple\")\n",
    "plt.barh(p_i, p_v, color=\"orange\")\n",
    "plt.barh(y_i, y_v, color=\"yellow\")\n",
    "plt.barh(g_i, g_v, color=\"green\")\n",
    "plt.barh(r_i, r_v, color=\"red\")\n",
    "plt.barh(b_i, b_v, color=\"blue\")\n",
    "plt.title('SNP Importance XGBoost Seed Coat Colour')\n",
    "plt.ylabel('SNP Label')\n",
    "plt.xlabel('Relative F_Score (GAIN)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RL Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt_vcf, ho_vcf, tt_pheno, ho_pheno = new_prep_data(\"SCC_Merged_filtered.csv_train_testQTL_SNPS.csv\", \"SCC_Merged_filtered.csv_holdoutQTL_SNPS.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if need or have new holdout data etc.\n",
    "ohe = pickle.load(open(\"SCC_QTL_ohe.dat\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tt_vcf.shape)\n",
    "tt_vcf = ohe.transform(tt_vcf)\n",
    "print(tt_vcf.shape)\n",
    "print(ho_vcf.shape)\n",
    "ho_vcf = ohe.transform(ho_vcf)\n",
    "print(ho_vcf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tt_vcf.shape)\n",
    "print(tt_pheno.shape)\n",
    "print(ho_vcf.shape)\n",
    "print(ho_pheno.shape)\n",
    "seed = randint(0,5000)\n",
    " #if optimised in same session, other enter manually below\n",
    "#this function should average out 10 folds and training, with inital params optimised\n",
    "#average accuracy and std should be calculated along with a nice AUROC graph of train/test models\n",
    "#best model should be extracted for use on holdout set\n",
    "model = RandomForestClassifier(n_estimators=100, random_state=seed, max_features = 'sqrt',n_jobs=1, verbose = 1)\n",
    "best_model = eval_k_fold(model, tt_vcf, tt_pheno, 10, ho_vcf, ho_pheno)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN Model (Based upon DL Primer & GMStools)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt_vcf, ho_vcf, tt_pheno, ho_pheno = new_prep_data(\"SCC_Merged_filtered.csv_train_testQTL_SNPS.csv\", \"SCC_Merged_filtered.csv_holdoutQTL_SNPS.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe = pickle.load(open(\"SCC_QTL_ohe.dat\", \"rb\"))\n",
    "tt_vcf = ohe.transform(tt_vcf)\n",
    "print(tt_vcf.shape)\n",
    "print(ho_vcf.shape)\n",
    "ho_vcf = ohe.transform(ho_vcf)\n",
    "print(ho_vcf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##how to mlb both tt and ho for same scheme? do i even need to?\n",
    "\n",
    "mlb = MultiLabelBinarizer()\n",
    "mlb = mlb.fit(tt_pheno)\n",
    "##print(tt_pheno.shape)\n",
    "#print(ho_pheno.shape)\n",
    "#tt_pheno = mlb.transform(tt_pheno)\n",
    "#print(tt_pheno.shape)\n",
    "#ho_pheno = mlb.transform(ho_pheno)\n",
    "#print(ho_pheno.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_CNN_model(x_len):    \n",
    "    #del model\n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(filters=10, kernel_size=10, \n",
    "                     input_shape=(x_len, 1)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Conv1D(filters=8, kernel_size=8, \n",
    "                     input_shape=(10, 1)))\n",
    "    model.add(Activation('linear'))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Conv1D(filters=6, kernel_size=6, \n",
    "                     input_shape=(8, 1)))\n",
    "    model.add(Activation('linear'))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(24, activation='linear'))\n",
    "    model.add(Dense(16, activation='linear'))\n",
    "    model.add(Dense(8, activation='linear'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(4, activation='softmax'))\n",
    "    opt = tf.keras.optimizers.Adamax(learning_rate=0.003)#, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name=\"Adamax\"\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "\n",
    "    print(model.summary())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_cnn(x,y,k,mlb):\n",
    "    cv = StratifiedKFold(n_splits=k,shuffle=False)\n",
    "    best_model = []\n",
    "    results = []\n",
    "    highest = 0\n",
    "    i = 1\n",
    "    for train,test in cv.split(x,y):\n",
    "        print(y.shape)\n",
    "        print(y[train])\n",
    "        if(i==1):\n",
    "            y = mlb.transform(y)\n",
    "            print(y.shape)\n",
    "            print(y[train])\n",
    "        x = x.reshape(x.shape[0], x.shape[1], 1)\n",
    "        model = build_CNN_model(x[train].shape[1])\n",
    "        bs = ((x[train].shape[0])/20)\n",
    "        bs = round(bs)\n",
    "        history = model.fit(x[train], y[train], validation_data=(x[test], y[test]), epochs=100, batch_size=bs)\n",
    "        _, accuracy = model.evaluate(x[test], y[test], batch_size=bs, verbose=0)\n",
    "        accuracy = accuracy *100\n",
    "        print(\"accuracy for model \" + str(i) + \" is \" + str(accuracy))\n",
    "        if(accuracy > highest):\n",
    "            highest = accuracy\n",
    "            best_model = model\n",
    "        results.append(accuracy)\n",
    "        del model\n",
    "        i = i + 1\n",
    "    print(\"Training Testing Accuracy: %.2f%% (%.2f%%)\" % (np.mean(results), np.std(results))) \n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_CNN = eval_cnn(tt_vcf, tt_pheno, 10, mlb)\n",
    "#import pickle\n",
    "#pickle.dump(best_CNN, open(\"SCC_CNN_model.pickle.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = ((ho_vcf.shape[0])/40)\n",
    "bs = round(bs)\n",
    "ho_vcf = ho_vcf.reshape(ho_vcf.shape[0], ho_vcf.shape[1],1)\n",
    "ho_pheno = mlb.transform(ho_pheno)\n",
    "_, accuracy = best_CNN.evaluate(ho_vcf, ho_pheno, batch_size=bs, verbose=0)\n",
    "print(\"Holdout accuracy is \" + str(accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vcf = vcf.reshape(vcf.shape[0], vcf.shape[1], 1)\n",
    "pheno = pheno.reshape(pheno.shape[0], pheno.shape[1])\n",
    "estimator = KerasClassifier(build_fn=CNN_model, epochs=10, batch_size=32, verbose=1)\n",
    "kfold = KFold(n_splits=10, shuffle=True)\n",
    "results = cross_val_score(estimator, vcf, pheno, cv=kfold)\n",
    "print(\"Baseline: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DNN (based off yield prediction paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "(620,)\n",
      "(620, 1)\n",
      "50000\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "(155,)\n",
      "(155, 1)\n",
      "50000\n",
      "(620, 43568)\n",
      "(155, 43568)\n"
     ]
    }
   ],
   "source": [
    "tt_vcf, ho_vcf, tt_pheno, ho_pheno = new_prep_data(\"SCC_Merged_filtered.csv_train_testQTL_SNPS.csv\", \"SCC_Merged_filtered.csv_holdoutQTL_SNPS.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(620, 125766)\n",
      "(155, 43568)\n",
      "(155, 125766)\n"
     ]
    }
   ],
   "source": [
    "ohe = pickle.load(open(\"SCC_QTL_ohe.dat\", \"rb\"))\n",
    "tt_vcf = ohe.transform(tt_vcf)\n",
    "print(tt_vcf.shape)\n",
    "print(ho_vcf.shape)\n",
    "ho_vcf = ohe.transform(ho_vcf)\n",
    "print(ho_vcf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##how to mlb both tt and ho for same scheme? do i even need to?\n",
    "\n",
    "mlb = MultiLabelBinarizer()\n",
    "mlb = mlb.fit(tt_pheno)\n",
    "##print(tt_pheno.shape)\n",
    "#print(ho_pheno.shape)\n",
    "#tt_pheno = mlb.transform(tt_pheno)\n",
    "#print(tt_pheno.shape)\n",
    "#ho_pheno = mlb.transform(ho_pheno)\n",
    "#print(ho_pheno.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#My own DNN model based upon paper\n",
    "#del model #incase its stored a previous model\n",
    "#del history #for redoing shit\n",
    "\n",
    "#do batch size as 64\n",
    "#reduce the inputs by half when you read it in\n",
    "#add XGboost and RF to the one notebook\n",
    "def build_DNN_model(x_len):\n",
    "    model = Sequential()\n",
    "\n",
    "    #add first input layer, with no normalization\n",
    "    model.add(Dense(192, input_dim = x_len))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.03))\n",
    "    \n",
    "    model.add(Dense(128))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.02))\n",
    "    \n",
    "    model.add(Dense(64))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.01))\n",
    "    \n",
    "    model.add(Dense(32))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dense(16))\n",
    "    model.add(Activation('relu'))\n",
    "    #add output layer\n",
    "    model.add(Dense(4, activation='softmax'))\n",
    "    opt = tf.keras.optimizers.Adamax(learning_rate=0.003)#, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name=\"Adamax\"\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "\n",
    "    print(model.summary())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_dnn(x,y,k,mlb):\n",
    "    cv = StratifiedKFold(n_splits=k,shuffle=False)\n",
    "    best_model = []\n",
    "    results = []\n",
    "    highest = 0\n",
    "    i = 1\n",
    "    for train,test in cv.split(x,y):\n",
    "        print(y.shape)\n",
    "        print(y[train])\n",
    "        if(i==1):\n",
    "            y = mlb.transform(y)\n",
    "            print(y.shape)\n",
    "            print(y[train])\n",
    "        model = build_DNN_model(x[train].shape[1])\n",
    "        bs = ((x[train].shape[0])/20)\n",
    "        bs = round(bs)\n",
    "        history = model.fit(x[train], y[train], validation_data=(x[test], y[test]), epochs=100, batch_size=bs)\n",
    "        _, accuracy = model.evaluate(x[test], y[test], batch_size=bs, verbose=0)\n",
    "        accuracy = accuracy *100\n",
    "        print(\"accuracy for model \" + str(i) + \" is \" + str(accuracy))\n",
    "        if(accuracy > highest):\n",
    "            highest = accuracy\n",
    "            best_model = model\n",
    "        results.append(accuracy)\n",
    "        del model\n",
    "        i = i + 1\n",
    "    print(\"Training Testing Accuracy: %.2f%% (%.2f%%)\" % (np.mean(results), np.std(results))) \n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(620, 1)\n",
      "[[0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [3.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [1.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [3.]\n",
      " [3.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [0.]\n",
      " [1.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [1.]\n",
      " [3.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [3.]\n",
      " [2.]\n",
      " [1.]\n",
      " [2.]\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [1.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [2.]\n",
      " [3.]\n",
      " [1.]\n",
      " [2.]\n",
      " [2.]\n",
      " [0.]\n",
      " [2.]\n",
      " [2.]]\n",
      "(620, 4)\n",
      "[[1 0 0 0]\n",
      " [1 0 0 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 556 samples, validate on 64 samples\n",
      "Epoch 1/100\n",
      "556/556 [==============================] - 1s 2ms/sample - loss: 1.0918 - accuracy: 0.6115 - val_loss: 7.6100 - val_accuracy: 0.6562\n",
      "Epoch 2/100\n",
      "556/556 [==============================] - 0s 682us/sample - loss: 0.7346 - accuracy: 0.8076 - val_loss: 3.9427 - val_accuracy: 0.7188\n",
      "Epoch 3/100\n",
      "556/556 [==============================] - 0s 709us/sample - loss: 0.6069 - accuracy: 0.8165 - val_loss: 2.5926 - val_accuracy: 0.7188\n",
      "Epoch 4/100\n",
      "556/556 [==============================] - 0s 709us/sample - loss: 0.5307 - accuracy: 0.8435 - val_loss: 1.6134 - val_accuracy: 0.7500\n",
      "Epoch 5/100\n",
      "556/556 [==============================] - 0s 716us/sample - loss: 0.4827 - accuracy: 0.8489 - val_loss: 1.2122 - val_accuracy: 0.7500\n",
      "Epoch 6/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.4608 - accuracy: 0.8543 - val_loss: 0.7611 - val_accuracy: 0.8438\n",
      "Epoch 7/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.4407 - accuracy: 0.8615 - val_loss: 0.7103 - val_accuracy: 0.8125\n",
      "Epoch 8/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.4234 - accuracy: 0.8561 - val_loss: 0.5874 - val_accuracy: 0.8438\n",
      "Epoch 9/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.4112 - accuracy: 0.8723 - val_loss: 0.6181 - val_accuracy: 0.8438\n",
      "Epoch 10/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.3585 - accuracy: 0.8741 - val_loss: 0.5698 - val_accuracy: 0.8125\n",
      "Epoch 11/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.3314 - accuracy: 0.8795 - val_loss: 0.6877 - val_accuracy: 0.8438\n",
      "Epoch 12/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.3402 - accuracy: 0.8741 - val_loss: 0.5872 - val_accuracy: 0.8594\n",
      "Epoch 13/100\n",
      "556/556 [==============================] - 0s 667us/sample - loss: 0.3710 - accuracy: 0.8633 - val_loss: 0.5731 - val_accuracy: 0.8594\n",
      "Epoch 14/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.3276 - accuracy: 0.8849 - val_loss: 0.4547 - val_accuracy: 0.8594\n",
      "Epoch 15/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.3089 - accuracy: 0.8795 - val_loss: 0.4415 - val_accuracy: 0.8594\n",
      "Epoch 16/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.2782 - accuracy: 0.8885 - val_loss: 0.4655 - val_accuracy: 0.8594\n",
      "Epoch 17/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.2933 - accuracy: 0.8921 - val_loss: 0.5690 - val_accuracy: 0.8281\n",
      "Epoch 18/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.3048 - accuracy: 0.8813 - val_loss: 0.7487 - val_accuracy: 0.7344\n",
      "Epoch 19/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.2883 - accuracy: 0.8903 - val_loss: 0.5405 - val_accuracy: 0.8438\n",
      "Epoch 20/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.2712 - accuracy: 0.8957 - val_loss: 0.4739 - val_accuracy: 0.8594\n",
      "Epoch 21/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.2445 - accuracy: 0.9173 - val_loss: 0.5128 - val_accuracy: 0.8438\n",
      "Epoch 22/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.2615 - accuracy: 0.9047 - val_loss: 0.4915 - val_accuracy: 0.8594\n",
      "Epoch 23/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.2168 - accuracy: 0.9137 - val_loss: 0.5133 - val_accuracy: 0.8438\n",
      "Epoch 24/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.2199 - accuracy: 0.9263 - val_loss: 0.5164 - val_accuracy: 0.8438\n",
      "Epoch 25/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.2117 - accuracy: 0.9029 - val_loss: 0.8215 - val_accuracy: 0.6875\n",
      "Epoch 26/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.2169 - accuracy: 0.9281 - val_loss: 0.9544 - val_accuracy: 0.6562\n",
      "Epoch 27/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.2090 - accuracy: 0.9299 - val_loss: 0.5500 - val_accuracy: 0.8281\n",
      "Epoch 28/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.1802 - accuracy: 0.9406 - val_loss: 0.5541 - val_accuracy: 0.8438\n",
      "Epoch 29/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.1600 - accuracy: 0.9406 - val_loss: 0.4998 - val_accuracy: 0.8438\n",
      "Epoch 30/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.1590 - accuracy: 0.9442 - val_loss: 0.6135 - val_accuracy: 0.8281\n",
      "Epoch 31/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.1414 - accuracy: 0.9496 - val_loss: 0.5719 - val_accuracy: 0.8281\n",
      "Epoch 32/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.1361 - accuracy: 0.9532 - val_loss: 0.6370 - val_accuracy: 0.7969\n",
      "Epoch 33/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.1216 - accuracy: 0.9550 - val_loss: 0.5960 - val_accuracy: 0.7812\n",
      "Epoch 34/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.1261 - accuracy: 0.9586 - val_loss: 0.7820 - val_accuracy: 0.7344\n",
      "Epoch 35/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.1232 - accuracy: 0.9622 - val_loss: 0.6637 - val_accuracy: 0.8281\n",
      "Epoch 36/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.0987 - accuracy: 0.9622 - val_loss: 0.8202 - val_accuracy: 0.7188\n",
      "Epoch 37/100\n",
      "556/556 [==============================] - 0s 668us/sample - loss: 0.1770 - accuracy: 0.9317 - val_loss: 1.2512 - val_accuracy: 0.6094\n",
      "Epoch 38/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.1295 - accuracy: 0.9514 - val_loss: 1.3305 - val_accuracy: 0.8125\n",
      "Epoch 39/100\n",
      "556/556 [==============================] - 0s 668us/sample - loss: 0.1212 - accuracy: 0.9658 - val_loss: 0.9841 - val_accuracy: 0.7188\n",
      "Epoch 40/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0858 - accuracy: 0.9730 - val_loss: 0.6768 - val_accuracy: 0.8125\n",
      "Epoch 41/100\n",
      "556/556 [==============================] - 0s 680us/sample - loss: 0.0814 - accuracy: 0.9712 - val_loss: 0.7946 - val_accuracy: 0.8125\n",
      "Epoch 42/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0869 - accuracy: 0.9622 - val_loss: 0.6539 - val_accuracy: 0.8438\n",
      "Epoch 43/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0762 - accuracy: 0.9748 - val_loss: 0.8777 - val_accuracy: 0.7031\n",
      "Epoch 44/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0921 - accuracy: 0.9730 - val_loss: 0.6302 - val_accuracy: 0.8438\n",
      "Epoch 45/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0494 - accuracy: 0.9802 - val_loss: 0.9174 - val_accuracy: 0.8281\n",
      "Epoch 46/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.0714 - accuracy: 0.9766 - val_loss: 0.9462 - val_accuracy: 0.8281\n",
      "Epoch 47/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0638 - accuracy: 0.9784 - val_loss: 0.6836 - val_accuracy: 0.8438\n",
      "Epoch 48/100\n",
      "556/556 [==============================] - 0s 668us/sample - loss: 0.0320 - accuracy: 0.9874 - val_loss: 0.7223 - val_accuracy: 0.8438\n",
      "Epoch 49/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0497 - accuracy: 0.9856 - val_loss: 0.9114 - val_accuracy: 0.8281\n",
      "Epoch 50/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0927 - accuracy: 0.9658 - val_loss: 0.9914 - val_accuracy: 0.8125\n",
      "Epoch 51/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0557 - accuracy: 0.9820 - val_loss: 0.9604 - val_accuracy: 0.8438\n",
      "Epoch 52/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0512 - accuracy: 0.9838 - val_loss: 0.8821 - val_accuracy: 0.8438\n",
      "Epoch 53/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0493 - accuracy: 0.9874 - val_loss: 1.0808 - val_accuracy: 0.8281\n",
      "Epoch 54/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0473 - accuracy: 0.9856 - val_loss: 0.9398 - val_accuracy: 0.8281\n",
      "Epoch 55/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0471 - accuracy: 0.9820 - val_loss: 0.7208 - val_accuracy: 0.8281\n",
      "Epoch 56/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0322 - accuracy: 0.9928 - val_loss: 0.7505 - val_accuracy: 0.8438\n",
      "Epoch 57/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0844 - accuracy: 0.9712 - val_loss: 1.6459 - val_accuracy: 0.4688\n",
      "Epoch 58/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0579 - accuracy: 0.9784 - val_loss: 0.8052 - val_accuracy: 0.8438\n",
      "Epoch 59/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0574 - accuracy: 0.9748 - val_loss: 0.7103 - val_accuracy: 0.8438\n",
      "Epoch 60/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0357 - accuracy: 0.9910 - val_loss: 0.7442 - val_accuracy: 0.8594\n",
      "Epoch 61/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0313 - accuracy: 0.9892 - val_loss: 0.9663 - val_accuracy: 0.8281\n",
      "Epoch 62/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0404 - accuracy: 0.9820 - val_loss: 0.8975 - val_accuracy: 0.8438\n",
      "Epoch 63/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0192 - accuracy: 0.9964 - val_loss: 0.8547 - val_accuracy: 0.8438\n",
      "Epoch 64/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0184 - accuracy: 0.9982 - val_loss: 0.8759 - val_accuracy: 0.8438\n",
      "Epoch 65/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.0279 - accuracy: 0.9910 - val_loss: 1.0557 - val_accuracy: 0.8438\n",
      "Epoch 66/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0263 - accuracy: 0.9910 - val_loss: 0.9633 - val_accuracy: 0.7812\n",
      "Epoch 67/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0169 - accuracy: 0.9910 - val_loss: 0.9059 - val_accuracy: 0.8438\n",
      "Epoch 68/100\n",
      "556/556 [==============================] - 0s 670us/sample - loss: 0.0210 - accuracy: 0.9946 - val_loss: 1.0028 - val_accuracy: 0.8438\n",
      "Epoch 69/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0150 - accuracy: 0.9964 - val_loss: 0.9883 - val_accuracy: 0.8438\n",
      "Epoch 70/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0156 - accuracy: 0.9964 - val_loss: 0.9335 - val_accuracy: 0.8594\n",
      "Epoch 71/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.0212 - accuracy: 0.9928 - val_loss: 1.1027 - val_accuracy: 0.8438\n",
      "Epoch 72/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0176 - accuracy: 0.9946 - val_loss: 1.3295 - val_accuracy: 0.8281\n",
      "Epoch 73/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0368 - accuracy: 0.9892 - val_loss: 1.0318 - val_accuracy: 0.8438\n",
      "Epoch 74/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0129 - accuracy: 0.9964 - val_loss: 0.9663 - val_accuracy: 0.8594\n",
      "Epoch 75/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0156 - accuracy: 0.9946 - val_loss: 0.8654 - val_accuracy: 0.8594\n",
      "Epoch 76/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0140 - accuracy: 0.9982 - val_loss: 1.1896 - val_accuracy: 0.8281\n",
      "Epoch 77/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.0222 - accuracy: 0.9964 - val_loss: 0.8920 - val_accuracy: 0.8281\n",
      "Epoch 78/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0148 - accuracy: 0.9964 - val_loss: 1.1561 - val_accuracy: 0.8281\n",
      "Epoch 79/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0208 - accuracy: 0.9946 - val_loss: 1.0204 - val_accuracy: 0.8438\n",
      "Epoch 80/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0071 - accuracy: 0.9982 - val_loss: 1.0394 - val_accuracy: 0.8438\n",
      "Epoch 81/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0181 - accuracy: 0.9946 - val_loss: 1.1875 - val_accuracy: 0.8438\n",
      "Epoch 82/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0076 - accuracy: 1.0000 - val_loss: 1.0041 - val_accuracy: 0.8438\n",
      "Epoch 83/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0086 - accuracy: 1.0000 - val_loss: 1.1023 - val_accuracy: 0.8281\n",
      "Epoch 84/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.0102 - accuracy: 0.9964 - val_loss: 1.1413 - val_accuracy: 0.8438\n",
      "Epoch 85/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.0266 - accuracy: 0.9928 - val_loss: 1.2956 - val_accuracy: 0.6719\n",
      "Epoch 86/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0283 - accuracy: 0.9910 - val_loss: 0.9676 - val_accuracy: 0.8594\n",
      "Epoch 87/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0083 - accuracy: 0.9982 - val_loss: 1.2472 - val_accuracy: 0.8438\n",
      "Epoch 88/100\n",
      "556/556 [==============================] - 0s 675us/sample - loss: 0.0068 - accuracy: 1.0000 - val_loss: 1.2010 - val_accuracy: 0.8438\n",
      "Epoch 89/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0263 - accuracy: 0.9910 - val_loss: 0.8385 - val_accuracy: 0.8594\n",
      "Epoch 90/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.0284 - accuracy: 0.9946 - val_loss: 0.9024 - val_accuracy: 0.8594\n",
      "Epoch 91/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.0220 - accuracy: 0.9928 - val_loss: 1.3728 - val_accuracy: 0.8281\n",
      "Epoch 92/100\n",
      "556/556 [==============================] - 0s 657us/sample - loss: 0.0174 - accuracy: 0.9928 - val_loss: 1.0511 - val_accuracy: 0.8438\n",
      "Epoch 93/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0082 - accuracy: 0.9964 - val_loss: 1.0139 - val_accuracy: 0.8438\n",
      "Epoch 94/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0082 - accuracy: 0.9982 - val_loss: 0.9984 - val_accuracy: 0.8594\n",
      "Epoch 95/100\n",
      "556/556 [==============================] - 0s 667us/sample - loss: 0.0130 - accuracy: 0.9946 - val_loss: 0.9870 - val_accuracy: 0.8594\n",
      "Epoch 96/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0045 - accuracy: 0.9982 - val_loss: 1.1174 - val_accuracy: 0.8594\n",
      "Epoch 97/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.0163 - accuracy: 0.9964 - val_loss: 1.0772 - val_accuracy: 0.8594\n",
      "Epoch 98/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0170 - accuracy: 0.9910 - val_loss: 1.2117 - val_accuracy: 0.8438\n",
      "Epoch 99/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0123 - accuracy: 0.9964 - val_loss: 1.2674 - val_accuracy: 0.8438\n",
      "Epoch 100/100\n",
      "556/556 [==============================] - 0s 651us/sample - loss: 0.0242 - accuracy: 0.9892 - val_loss: 0.9428 - val_accuracy: 0.8594\n",
      "accuracy for model 1 is 85.9375\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 556 samples, validate on 64 samples\n",
      "Epoch 1/100\n",
      "556/556 [==============================] - 1s 2ms/sample - loss: 1.1068 - accuracy: 0.5144 - val_loss: 4.8471 - val_accuracy: 0.6562\n",
      "Epoch 2/100\n",
      "556/556 [==============================] - 0s 683us/sample - loss: 0.6570 - accuracy: 0.8040 - val_loss: 2.9236 - val_accuracy: 0.6875\n",
      "Epoch 3/100\n",
      "556/556 [==============================] - 0s 697us/sample - loss: 0.5339 - accuracy: 0.8327 - val_loss: 1.9781 - val_accuracy: 0.6875\n",
      "Epoch 4/100\n",
      "556/556 [==============================] - 0s 702us/sample - loss: 0.4441 - accuracy: 0.8669 - val_loss: 1.0490 - val_accuracy: 0.8125\n",
      "Epoch 5/100\n",
      "556/556 [==============================] - 0s 683us/sample - loss: 0.4366 - accuracy: 0.8687 - val_loss: 0.6883 - val_accuracy: 0.8125\n",
      "Epoch 6/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.4073 - accuracy: 0.8633 - val_loss: 0.5203 - val_accuracy: 0.8438\n",
      "Epoch 7/100\n",
      "556/556 [==============================] - 0s 682us/sample - loss: 0.3941 - accuracy: 0.8705 - val_loss: 0.5051 - val_accuracy: 0.8125\n",
      "Epoch 8/100\n",
      "556/556 [==============================] - 0s 700us/sample - loss: 0.3723 - accuracy: 0.8813 - val_loss: 0.5148 - val_accuracy: 0.8594\n",
      "Epoch 9/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.3614 - accuracy: 0.8687 - val_loss: 0.7381 - val_accuracy: 0.7812\n",
      "Epoch 10/100\n",
      "556/556 [==============================] - 0s 690us/sample - loss: 0.3528 - accuracy: 0.8687 - val_loss: 0.5072 - val_accuracy: 0.8281\n",
      "Epoch 11/100\n",
      "556/556 [==============================] - 0s 684us/sample - loss: 0.3569 - accuracy: 0.8687 - val_loss: 0.7423 - val_accuracy: 0.7969\n",
      "Epoch 12/100\n",
      "556/556 [==============================] - 0s 687us/sample - loss: 0.3185 - accuracy: 0.8813 - val_loss: 0.5575 - val_accuracy: 0.7969\n",
      "Epoch 13/100\n",
      "556/556 [==============================] - 0s 706us/sample - loss: 0.3241 - accuracy: 0.8777 - val_loss: 0.5018 - val_accuracy: 0.8438\n",
      "Epoch 14/100\n",
      "556/556 [==============================] - 0s 684us/sample - loss: 0.3087 - accuracy: 0.8849 - val_loss: 0.5151 - val_accuracy: 0.8438\n",
      "Epoch 15/100\n",
      "556/556 [==============================] - 0s 697us/sample - loss: 0.2710 - accuracy: 0.8939 - val_loss: 0.6523 - val_accuracy: 0.7969\n",
      "Epoch 16/100\n",
      "556/556 [==============================] - 0s 690us/sample - loss: 0.2679 - accuracy: 0.8903 - val_loss: 1.0092 - val_accuracy: 0.7656\n",
      "Epoch 17/100\n",
      "556/556 [==============================] - 0s 680us/sample - loss: 0.2609 - accuracy: 0.8903 - val_loss: 0.7818 - val_accuracy: 0.7969\n",
      "Epoch 18/100\n",
      "556/556 [==============================] - 0s 686us/sample - loss: 0.2336 - accuracy: 0.9119 - val_loss: 0.4865 - val_accuracy: 0.8438\n",
      "Epoch 19/100\n",
      "556/556 [==============================] - 0s 685us/sample - loss: 0.2330 - accuracy: 0.9029 - val_loss: 0.7209 - val_accuracy: 0.7812\n",
      "Epoch 20/100\n",
      "556/556 [==============================] - 0s 686us/sample - loss: 0.2208 - accuracy: 0.9155 - val_loss: 0.6116 - val_accuracy: 0.7969\n",
      "Epoch 21/100\n",
      "556/556 [==============================] - 0s 711us/sample - loss: 0.1989 - accuracy: 0.9353 - val_loss: 0.5581 - val_accuracy: 0.8438\n",
      "Epoch 22/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.2393 - accuracy: 0.9029 - val_loss: 0.6051 - val_accuracy: 0.8281\n",
      "Epoch 23/100\n",
      "556/556 [==============================] - 0s 691us/sample - loss: 0.1963 - accuracy: 0.9263 - val_loss: 0.7390 - val_accuracy: 0.8281\n",
      "Epoch 24/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.1598 - accuracy: 0.9406 - val_loss: 1.3224 - val_accuracy: 0.7500\n",
      "Epoch 25/100\n",
      "556/556 [==============================] - 0s 682us/sample - loss: 0.1764 - accuracy: 0.9335 - val_loss: 0.8092 - val_accuracy: 0.7969\n",
      "Epoch 26/100\n",
      "556/556 [==============================] - 0s 706us/sample - loss: 0.1660 - accuracy: 0.9478 - val_loss: 0.9620 - val_accuracy: 0.7969\n",
      "Epoch 27/100\n",
      "556/556 [==============================] - 0s 680us/sample - loss: 0.1596 - accuracy: 0.9514 - val_loss: 1.8362 - val_accuracy: 0.7812\n",
      "Epoch 28/100\n",
      "556/556 [==============================] - 0s 681us/sample - loss: 0.1615 - accuracy: 0.9317 - val_loss: 1.2848 - val_accuracy: 0.7656\n",
      "Epoch 29/100\n",
      "556/556 [==============================] - 0s 690us/sample - loss: 0.1245 - accuracy: 0.9550 - val_loss: 0.4993 - val_accuracy: 0.8125\n",
      "Epoch 30/100\n",
      "556/556 [==============================] - 0s 683us/sample - loss: 0.0929 - accuracy: 0.9730 - val_loss: 0.9513 - val_accuracy: 0.7656\n",
      "Epoch 31/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.1171 - accuracy: 0.9604 - val_loss: 1.1769 - val_accuracy: 0.7812\n",
      "Epoch 32/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0928 - accuracy: 0.9676 - val_loss: 0.8241 - val_accuracy: 0.7656\n",
      "Epoch 33/100\n",
      "556/556 [==============================] - 0s 694us/sample - loss: 0.1418 - accuracy: 0.9442 - val_loss: 1.1976 - val_accuracy: 0.7969\n",
      "Epoch 34/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.1107 - accuracy: 0.9586 - val_loss: 0.9132 - val_accuracy: 0.8125\n",
      "Epoch 35/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0780 - accuracy: 0.9658 - val_loss: 0.9411 - val_accuracy: 0.7969\n",
      "Epoch 36/100\n",
      "556/556 [==============================] - 0s 684us/sample - loss: 0.0753 - accuracy: 0.9784 - val_loss: 0.8168 - val_accuracy: 0.8125\n",
      "Epoch 37/100\n",
      "556/556 [==============================] - 0s 711us/sample - loss: 0.0802 - accuracy: 0.9748 - val_loss: 1.3392 - val_accuracy: 0.7969\n",
      "Epoch 38/100\n",
      "556/556 [==============================] - 0s 684us/sample - loss: 0.0841 - accuracy: 0.9604 - val_loss: 0.6947 - val_accuracy: 0.8281\n",
      "Epoch 39/100\n",
      "556/556 [==============================] - 0s 681us/sample - loss: 0.0579 - accuracy: 0.9766 - val_loss: 0.8089 - val_accuracy: 0.7656\n",
      "Epoch 40/100\n",
      "556/556 [==============================] - 0s 717us/sample - loss: 0.0784 - accuracy: 0.9712 - val_loss: 0.9094 - val_accuracy: 0.8125\n",
      "Epoch 41/100\n",
      "556/556 [==============================] - 0s 672us/sample - loss: 0.0514 - accuracy: 0.9874 - val_loss: 0.9612 - val_accuracy: 0.7969\n",
      "Epoch 42/100\n",
      "556/556 [==============================] - 0s 700us/sample - loss: 0.0450 - accuracy: 0.9892 - val_loss: 0.6851 - val_accuracy: 0.8594\n",
      "Epoch 43/100\n",
      "556/556 [==============================] - 0s 685us/sample - loss: 0.0412 - accuracy: 0.9892 - val_loss: 0.8313 - val_accuracy: 0.8125\n",
      "Epoch 44/100\n",
      "556/556 [==============================] - 0s 709us/sample - loss: 0.0334 - accuracy: 0.9928 - val_loss: 0.8579 - val_accuracy: 0.7969\n",
      "Epoch 45/100\n",
      "556/556 [==============================] - 0s 688us/sample - loss: 0.0458 - accuracy: 0.9874 - val_loss: 1.2430 - val_accuracy: 0.7656\n",
      "Epoch 46/100\n",
      "556/556 [==============================] - 0s 712us/sample - loss: 0.0477 - accuracy: 0.9820 - val_loss: 1.2446 - val_accuracy: 0.7812\n",
      "Epoch 47/100\n",
      "556/556 [==============================] - 0s 678us/sample - loss: 0.0572 - accuracy: 0.9766 - val_loss: 1.2020 - val_accuracy: 0.7656\n",
      "Epoch 48/100\n",
      "556/556 [==============================] - 0s 684us/sample - loss: 0.0508 - accuracy: 0.9784 - val_loss: 1.1782 - val_accuracy: 0.7500\n",
      "Epoch 49/100\n",
      "556/556 [==============================] - 0s 701us/sample - loss: 0.0444 - accuracy: 0.9838 - val_loss: 0.8026 - val_accuracy: 0.8438\n",
      "Epoch 50/100\n",
      "556/556 [==============================] - 0s 673us/sample - loss: 0.0616 - accuracy: 0.9766 - val_loss: 1.0547 - val_accuracy: 0.7344\n",
      "Epoch 51/100\n",
      "556/556 [==============================] - 0s 687us/sample - loss: 0.0426 - accuracy: 0.9874 - val_loss: 0.6841 - val_accuracy: 0.7812\n",
      "Epoch 52/100\n",
      "556/556 [==============================] - 0s 712us/sample - loss: 0.0479 - accuracy: 0.9838 - val_loss: 0.6057 - val_accuracy: 0.8438\n",
      "Epoch 53/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.0389 - accuracy: 0.9928 - val_loss: 0.7918 - val_accuracy: 0.7969\n",
      "Epoch 54/100\n",
      "556/556 [==============================] - 0s 690us/sample - loss: 0.0412 - accuracy: 0.9874 - val_loss: 0.8431 - val_accuracy: 0.8125\n",
      "Epoch 55/100\n",
      "556/556 [==============================] - 0s 700us/sample - loss: 0.0240 - accuracy: 0.9928 - val_loss: 0.7834 - val_accuracy: 0.8438\n",
      "Epoch 56/100\n",
      "556/556 [==============================] - 0s 685us/sample - loss: 0.0245 - accuracy: 0.9946 - val_loss: 0.8308 - val_accuracy: 0.8281\n",
      "Epoch 57/100\n",
      "556/556 [==============================] - 0s 694us/sample - loss: 0.0525 - accuracy: 0.9802 - val_loss: 0.7455 - val_accuracy: 0.8281\n",
      "Epoch 58/100\n",
      "556/556 [==============================] - 0s 698us/sample - loss: 0.0441 - accuracy: 0.9856 - val_loss: 0.7036 - val_accuracy: 0.8281\n",
      "Epoch 59/100\n",
      "556/556 [==============================] - 0s 697us/sample - loss: 0.0215 - accuracy: 0.9946 - val_loss: 0.7180 - val_accuracy: 0.7969\n",
      "Epoch 60/100\n",
      "556/556 [==============================] - 0s 718us/sample - loss: 0.0423 - accuracy: 0.9856 - val_loss: 1.5514 - val_accuracy: 0.7969\n",
      "Epoch 61/100\n",
      "556/556 [==============================] - 0s 688us/sample - loss: 0.0639 - accuracy: 0.9784 - val_loss: 1.1337 - val_accuracy: 0.7344\n",
      "Epoch 62/100\n",
      "556/556 [==============================] - 0s 711us/sample - loss: 0.0178 - accuracy: 0.9982 - val_loss: 1.0121 - val_accuracy: 0.7656\n",
      "Epoch 63/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0265 - accuracy: 0.9928 - val_loss: 1.3187 - val_accuracy: 0.7812\n",
      "Epoch 64/100\n",
      "556/556 [==============================] - 0s 703us/sample - loss: 0.0201 - accuracy: 0.9964 - val_loss: 1.1268 - val_accuracy: 0.7969\n",
      "Epoch 65/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "556/556 [==============================] - 0s 683us/sample - loss: 0.0166 - accuracy: 0.9982 - val_loss: 0.8367 - val_accuracy: 0.8281\n",
      "Epoch 66/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0239 - accuracy: 0.9910 - val_loss: 0.9738 - val_accuracy: 0.8125\n",
      "Epoch 67/100\n",
      "556/556 [==============================] - 0s 686us/sample - loss: 0.0177 - accuracy: 0.9946 - val_loss: 1.1550 - val_accuracy: 0.7656\n",
      "Epoch 68/100\n",
      "556/556 [==============================] - 0s 669us/sample - loss: 0.0164 - accuracy: 0.9946 - val_loss: 0.6174 - val_accuracy: 0.8438\n",
      "Epoch 69/100\n",
      "556/556 [==============================] - 0s 719us/sample - loss: 0.0220 - accuracy: 0.9910 - val_loss: 0.9576 - val_accuracy: 0.7812\n",
      "Epoch 70/100\n",
      "556/556 [==============================] - 0s 669us/sample - loss: 0.0121 - accuracy: 0.9964 - val_loss: 1.0086 - val_accuracy: 0.7812\n",
      "Epoch 71/100\n",
      "556/556 [==============================] - 0s 671us/sample - loss: 0.0124 - accuracy: 0.9982 - val_loss: 0.8638 - val_accuracy: 0.8281\n",
      "Epoch 72/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.0078 - accuracy: 1.0000 - val_loss: 0.9020 - val_accuracy: 0.8125\n",
      "Epoch 73/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0083 - accuracy: 1.0000 - val_loss: 0.9969 - val_accuracy: 0.7969\n",
      "Epoch 74/100\n",
      "556/556 [==============================] - 0s 702us/sample - loss: 0.0150 - accuracy: 0.9964 - val_loss: 0.8019 - val_accuracy: 0.8438\n",
      "Epoch 75/100\n",
      "556/556 [==============================] - 0s 681us/sample - loss: 0.0127 - accuracy: 0.9982 - val_loss: 0.9376 - val_accuracy: 0.8125\n",
      "Epoch 76/100\n",
      "556/556 [==============================] - 0s 710us/sample - loss: 0.0284 - accuracy: 0.9910 - val_loss: 0.7054 - val_accuracy: 0.8281\n",
      "Epoch 77/100\n",
      "556/556 [==============================] - 0s 673us/sample - loss: 0.0324 - accuracy: 0.9910 - val_loss: 0.9906 - val_accuracy: 0.7812\n",
      "Epoch 78/100\n",
      "556/556 [==============================] - 0s 712us/sample - loss: 0.0283 - accuracy: 0.9946 - val_loss: 0.9335 - val_accuracy: 0.7656\n",
      "Epoch 79/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0164 - accuracy: 0.9964 - val_loss: 1.5876 - val_accuracy: 0.7812\n",
      "Epoch 80/100\n",
      "556/556 [==============================] - 0s 690us/sample - loss: 0.0283 - accuracy: 0.9874 - val_loss: 1.2004 - val_accuracy: 0.7344\n",
      "Epoch 81/100\n",
      "556/556 [==============================] - 0s 711us/sample - loss: 0.0091 - accuracy: 0.9964 - val_loss: 1.3963 - val_accuracy: 0.7969\n",
      "Epoch 82/100\n",
      "556/556 [==============================] - 0s 691us/sample - loss: 0.0056 - accuracy: 1.0000 - val_loss: 0.8557 - val_accuracy: 0.8438\n",
      "Epoch 83/100\n",
      "556/556 [==============================] - 0s 683us/sample - loss: 0.0030 - accuracy: 1.0000 - val_loss: 1.0188 - val_accuracy: 0.8281\n",
      "Epoch 84/100\n",
      "556/556 [==============================] - 0s 679us/sample - loss: 0.0076 - accuracy: 0.9982 - val_loss: 1.0229 - val_accuracy: 0.8281\n",
      "Epoch 85/100\n",
      "556/556 [==============================] - 0s 682us/sample - loss: 0.0154 - accuracy: 0.9964 - val_loss: 0.8428 - val_accuracy: 0.8125\n",
      "Epoch 86/100\n",
      "556/556 [==============================] - 0s 676us/sample - loss: 0.0106 - accuracy: 1.0000 - val_loss: 1.2311 - val_accuracy: 0.7812\n",
      "Epoch 87/100\n",
      "556/556 [==============================] - 0s 685us/sample - loss: 0.0148 - accuracy: 0.9928 - val_loss: 1.7308 - val_accuracy: 0.7812\n",
      "Epoch 88/100\n",
      "556/556 [==============================] - 0s 714us/sample - loss: 0.0133 - accuracy: 0.9964 - val_loss: 1.4637 - val_accuracy: 0.7969\n",
      "Epoch 89/100\n",
      "556/556 [==============================] - 0s 694us/sample - loss: 0.0114 - accuracy: 0.9946 - val_loss: 0.8521 - val_accuracy: 0.8438\n",
      "Epoch 90/100\n",
      "556/556 [==============================] - 0s 681us/sample - loss: 0.0118 - accuracy: 0.9964 - val_loss: 1.6450 - val_accuracy: 0.7812\n",
      "Epoch 91/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.0287 - accuracy: 0.9928 - val_loss: 0.7960 - val_accuracy: 0.7812\n",
      "Epoch 92/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.0112 - accuracy: 0.9964 - val_loss: 0.8936 - val_accuracy: 0.8125\n",
      "Epoch 93/100\n",
      "556/556 [==============================] - 0s 677us/sample - loss: 0.0100 - accuracy: 1.0000 - val_loss: 1.0327 - val_accuracy: 0.8125\n",
      "Epoch 94/100\n",
      "556/556 [==============================] - 0s 667us/sample - loss: 0.0319 - accuracy: 0.9892 - val_loss: 1.2170 - val_accuracy: 0.7812\n",
      "Epoch 95/100\n",
      "556/556 [==============================] - 0s 700us/sample - loss: 0.0340 - accuracy: 0.9892 - val_loss: 1.3197 - val_accuracy: 0.7656\n",
      "Epoch 96/100\n",
      "556/556 [==============================] - 0s 678us/sample - loss: 0.0338 - accuracy: 0.9856 - val_loss: 0.9206 - val_accuracy: 0.7969\n",
      "Epoch 97/100\n",
      "556/556 [==============================] - 0s 681us/sample - loss: 0.0259 - accuracy: 0.9910 - val_loss: 1.2329 - val_accuracy: 0.7969\n",
      "Epoch 98/100\n",
      "556/556 [==============================] - 0s 683us/sample - loss: 0.0272 - accuracy: 0.9910 - val_loss: 1.3082 - val_accuracy: 0.7500\n",
      "Epoch 99/100\n",
      "556/556 [==============================] - 0s 674us/sample - loss: 0.0110 - accuracy: 0.9946 - val_loss: 0.8751 - val_accuracy: 0.8281\n",
      "Epoch 100/100\n",
      "556/556 [==============================] - 0s 682us/sample - loss: 0.0160 - accuracy: 0.9946 - val_loss: 1.2840 - val_accuracy: 0.7812\n",
      "accuracy for model 2 is 78.125\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 556 samples, validate on 64 samples\n",
      "Epoch 1/100\n",
      "556/556 [==============================] - 1s 2ms/sample - loss: 0.9452 - accuracy: 0.6709 - val_loss: 2.3267 - val_accuracy: 0.6562\n",
      "Epoch 2/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.6639 - accuracy: 0.8165 - val_loss: 0.8445 - val_accuracy: 0.8438\n",
      "Epoch 3/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.5709 - accuracy: 0.8453 - val_loss: 0.6437 - val_accuracy: 0.8438\n",
      "Epoch 4/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.5193 - accuracy: 0.8525 - val_loss: 0.5534 - val_accuracy: 0.8438\n",
      "Epoch 5/100\n",
      "556/556 [==============================] - 0s 655us/sample - loss: 0.4696 - accuracy: 0.8543 - val_loss: 1.1858 - val_accuracy: 0.5156\n",
      "Epoch 6/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.4832 - accuracy: 0.8543 - val_loss: 0.6301 - val_accuracy: 0.8125\n",
      "Epoch 7/100\n",
      "556/556 [==============================] - 0s 655us/sample - loss: 0.4258 - accuracy: 0.8669 - val_loss: 0.5962 - val_accuracy: 0.8594\n",
      "Epoch 8/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.3770 - accuracy: 0.8723 - val_loss: 0.5271 - val_accuracy: 0.8438\n",
      "Epoch 9/100\n",
      "556/556 [==============================] - 0s 667us/sample - loss: 0.3583 - accuracy: 0.8831 - val_loss: 0.5650 - val_accuracy: 0.8594\n",
      "Epoch 10/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.3478 - accuracy: 0.8759 - val_loss: 0.5354 - val_accuracy: 0.8438\n",
      "Epoch 11/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.3127 - accuracy: 0.8741 - val_loss: 0.4988 - val_accuracy: 0.8438\n",
      "Epoch 12/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.2956 - accuracy: 0.8957 - val_loss: 0.5263 - val_accuracy: 0.8594\n",
      "Epoch 13/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.2956 - accuracy: 0.8993 - val_loss: 0.4887 - val_accuracy: 0.8438\n",
      "Epoch 14/100\n",
      "556/556 [==============================] - 0s 651us/sample - loss: 0.2765 - accuracy: 0.8939 - val_loss: 0.5436 - val_accuracy: 0.8750\n",
      "Epoch 15/100\n",
      "556/556 [==============================] - 0s 653us/sample - loss: 0.2522 - accuracy: 0.8993 - val_loss: 0.4916 - val_accuracy: 0.8750\n",
      "Epoch 16/100\n",
      "556/556 [==============================] - 0s 656us/sample - loss: 0.2544 - accuracy: 0.9047 - val_loss: 0.5090 - val_accuracy: 0.8594\n",
      "Epoch 17/100\n",
      "556/556 [==============================] - 0s 652us/sample - loss: 0.2036 - accuracy: 0.9209 - val_loss: 0.4417 - val_accuracy: 0.8281\n",
      "Epoch 18/100\n",
      "556/556 [==============================] - 0s 653us/sample - loss: 0.2083 - accuracy: 0.9191 - val_loss: 0.5556 - val_accuracy: 0.7500\n",
      "Epoch 19/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.2097 - accuracy: 0.9299 - val_loss: 0.4805 - val_accuracy: 0.8438\n",
      "Epoch 20/100\n",
      "556/556 [==============================] - 0s 653us/sample - loss: 0.1720 - accuracy: 0.9388 - val_loss: 0.5773 - val_accuracy: 0.8594\n",
      "Epoch 21/100\n",
      "556/556 [==============================] - 0s 651us/sample - loss: 0.1846 - accuracy: 0.9335 - val_loss: 0.6392 - val_accuracy: 0.8906\n",
      "Epoch 22/100\n",
      "556/556 [==============================] - 0s 652us/sample - loss: 0.1505 - accuracy: 0.9478 - val_loss: 0.5000 - val_accuracy: 0.8438\n",
      "Epoch 23/100\n",
      "556/556 [==============================] - 0s 652us/sample - loss: 0.1210 - accuracy: 0.9604 - val_loss: 0.4910 - val_accuracy: 0.8438\n",
      "Epoch 24/100\n",
      "556/556 [==============================] - 0s 652us/sample - loss: 0.1187 - accuracy: 0.9568 - val_loss: 0.5362 - val_accuracy: 0.8438\n",
      "Epoch 25/100\n",
      "556/556 [==============================] - 0s 651us/sample - loss: 0.1367 - accuracy: 0.9478 - val_loss: 0.4290 - val_accuracy: 0.8594\n",
      "Epoch 26/100\n",
      "556/556 [==============================] - 0s 651us/sample - loss: 0.1172 - accuracy: 0.9586 - val_loss: 0.5726 - val_accuracy: 0.8594\n",
      "Epoch 27/100\n",
      "556/556 [==============================] - 0s 653us/sample - loss: 0.1043 - accuracy: 0.9640 - val_loss: 0.5201 - val_accuracy: 0.8594\n",
      "Epoch 28/100\n",
      "556/556 [==============================] - 0s 666us/sample - loss: 0.0909 - accuracy: 0.9766 - val_loss: 0.5018 - val_accuracy: 0.8125\n",
      "Epoch 29/100\n",
      "556/556 [==============================] - 0s 676us/sample - loss: 0.1028 - accuracy: 0.9694 - val_loss: 0.6499 - val_accuracy: 0.8906\n",
      "Epoch 30/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0873 - accuracy: 0.9658 - val_loss: 0.6529 - val_accuracy: 0.8906\n",
      "Epoch 31/100\n",
      "556/556 [==============================] - 0s 676us/sample - loss: 0.0812 - accuracy: 0.9748 - val_loss: 0.6591 - val_accuracy: 0.8594\n",
      "Epoch 32/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0781 - accuracy: 0.9748 - val_loss: 0.6139 - val_accuracy: 0.8906\n",
      "Epoch 33/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0802 - accuracy: 0.9694 - val_loss: 0.8939 - val_accuracy: 0.8906\n",
      "Epoch 34/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0918 - accuracy: 0.9658 - val_loss: 0.4909 - val_accuracy: 0.8438\n",
      "Epoch 35/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0660 - accuracy: 0.9820 - val_loss: 0.6490 - val_accuracy: 0.8906\n",
      "Epoch 36/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0495 - accuracy: 0.9892 - val_loss: 0.6228 - val_accuracy: 0.8281\n",
      "Epoch 37/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0486 - accuracy: 0.9892 - val_loss: 0.6624 - val_accuracy: 0.8438\n",
      "Epoch 38/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.0345 - accuracy: 0.9928 - val_loss: 0.7089 - val_accuracy: 0.8906\n",
      "Epoch 39/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0313 - accuracy: 0.9928 - val_loss: 0.6716 - val_accuracy: 0.8906\n",
      "Epoch 40/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0555 - accuracy: 0.9802 - val_loss: 0.7362 - val_accuracy: 0.8594\n",
      "Epoch 41/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0329 - accuracy: 0.9910 - val_loss: 0.6773 - val_accuracy: 0.8750\n",
      "Epoch 42/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0478 - accuracy: 0.9856 - val_loss: 0.7223 - val_accuracy: 0.8750\n",
      "Epoch 43/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0393 - accuracy: 0.9928 - val_loss: 0.6747 - val_accuracy: 0.8750\n",
      "Epoch 44/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0186 - accuracy: 0.9946 - val_loss: 0.7372 - val_accuracy: 0.8750\n",
      "Epoch 45/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0471 - accuracy: 0.9820 - val_loss: 0.6608 - val_accuracy: 0.8906\n",
      "Epoch 46/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0237 - accuracy: 0.9946 - val_loss: 0.6744 - val_accuracy: 0.8906\n",
      "Epoch 47/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0298 - accuracy: 0.9928 - val_loss: 0.6050 - val_accuracy: 0.8594\n",
      "Epoch 48/100\n",
      "556/556 [==============================] - 0s 658us/sample - loss: 0.0274 - accuracy: 0.9910 - val_loss: 0.8325 - val_accuracy: 0.8906\n",
      "Epoch 49/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0222 - accuracy: 0.9946 - val_loss: 0.7988 - val_accuracy: 0.8750\n",
      "Epoch 50/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0321 - accuracy: 0.9946 - val_loss: 0.9322 - val_accuracy: 0.8594\n",
      "Epoch 51/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0302 - accuracy: 0.9892 - val_loss: 0.8497 - val_accuracy: 0.7969\n",
      "Epoch 52/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0239 - accuracy: 0.9928 - val_loss: 0.7673 - val_accuracy: 0.8438\n",
      "Epoch 53/100\n",
      "556/556 [==============================] - 0s 659us/sample - loss: 0.0209 - accuracy: 0.9946 - val_loss: 0.9626 - val_accuracy: 0.8906\n",
      "Epoch 54/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0113 - accuracy: 0.9982 - val_loss: 0.8201 - val_accuracy: 0.8125\n",
      "Epoch 55/100\n",
      "556/556 [==============================] - 0s 657us/sample - loss: 0.0108 - accuracy: 0.9982 - val_loss: 0.8658 - val_accuracy: 0.8750\n",
      "Epoch 56/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0089 - accuracy: 1.0000 - val_loss: 0.8416 - val_accuracy: 0.8906\n",
      "Epoch 57/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.0141 - accuracy: 0.9982 - val_loss: 0.7848 - val_accuracy: 0.8906\n",
      "Epoch 58/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "556/556 [==============================] - 0s 666us/sample - loss: 0.0237 - accuracy: 0.9946 - val_loss: 0.7050 - val_accuracy: 0.8594\n",
      "Epoch 59/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0444 - accuracy: 0.9856 - val_loss: 0.6317 - val_accuracy: 0.8906\n",
      "Epoch 60/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0271 - accuracy: 0.9946 - val_loss: 0.8931 - val_accuracy: 0.7656\n",
      "Epoch 61/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0129 - accuracy: 0.9964 - val_loss: 0.7688 - val_accuracy: 0.8906\n",
      "Epoch 62/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0240 - accuracy: 0.9928 - val_loss: 1.0962 - val_accuracy: 0.8750\n",
      "Epoch 63/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0183 - accuracy: 0.9928 - val_loss: 0.8956 - val_accuracy: 0.8750\n",
      "Epoch 64/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0188 - accuracy: 0.9928 - val_loss: 0.8772 - val_accuracy: 0.8906\n",
      "Epoch 65/100\n",
      "556/556 [==============================] - 0s 661us/sample - loss: 0.0170 - accuracy: 0.9928 - val_loss: 0.7589 - val_accuracy: 0.8438\n",
      "Epoch 66/100\n",
      "556/556 [==============================] - 0s 657us/sample - loss: 0.0182 - accuracy: 0.9982 - val_loss: 0.7988 - val_accuracy: 0.7969\n",
      "Epoch 67/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0175 - accuracy: 0.9946 - val_loss: 0.7870 - val_accuracy: 0.8438\n",
      "Epoch 68/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0233 - accuracy: 0.9928 - val_loss: 0.6735 - val_accuracy: 0.8750\n",
      "Epoch 69/100\n",
      "556/556 [==============================] - 0s 659us/sample - loss: 0.0211 - accuracy: 0.9928 - val_loss: 0.7844 - val_accuracy: 0.8438\n",
      "Epoch 70/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0216 - accuracy: 0.9910 - val_loss: 0.7581 - val_accuracy: 0.8906\n",
      "Epoch 71/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0235 - accuracy: 0.9928 - val_loss: 0.8111 - val_accuracy: 0.8281\n",
      "Epoch 72/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0308 - accuracy: 0.9892 - val_loss: 0.7907 - val_accuracy: 0.8906\n",
      "Epoch 73/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0132 - accuracy: 0.9982 - val_loss: 1.0880 - val_accuracy: 0.8125\n",
      "Epoch 74/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0109 - accuracy: 0.9982 - val_loss: 0.8886 - val_accuracy: 0.8594\n",
      "Epoch 75/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0088 - accuracy: 1.0000 - val_loss: 0.9358 - val_accuracy: 0.7656\n",
      "Epoch 76/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0080 - accuracy: 0.9982 - val_loss: 1.0000 - val_accuracy: 0.8594\n",
      "Epoch 77/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0071 - accuracy: 0.9982 - val_loss: 1.0044 - val_accuracy: 0.8750\n",
      "Epoch 78/100\n",
      "556/556 [==============================] - 0s 659us/sample - loss: 0.0178 - accuracy: 0.9946 - val_loss: 1.1858 - val_accuracy: 0.8594\n",
      "Epoch 79/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0167 - accuracy: 0.9946 - val_loss: 0.9474 - val_accuracy: 0.8438\n",
      "Epoch 80/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0102 - accuracy: 0.9982 - val_loss: 1.0030 - val_accuracy: 0.8594\n",
      "Epoch 81/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0158 - accuracy: 0.9928 - val_loss: 1.0552 - val_accuracy: 0.8594\n",
      "Epoch 82/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0111 - accuracy: 0.9982 - val_loss: 0.9022 - val_accuracy: 0.8750\n",
      "Epoch 83/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0091 - accuracy: 0.9982 - val_loss: 0.9604 - val_accuracy: 0.8125\n",
      "Epoch 84/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0180 - accuracy: 0.9928 - val_loss: 1.1881 - val_accuracy: 0.8750\n",
      "Epoch 85/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0163 - accuracy: 0.9946 - val_loss: 1.1969 - val_accuracy: 0.8594\n",
      "Epoch 86/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0179 - accuracy: 0.9964 - val_loss: 0.9501 - val_accuracy: 0.8281\n",
      "Epoch 87/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0244 - accuracy: 0.9892 - val_loss: 1.4386 - val_accuracy: 0.6562\n",
      "Epoch 88/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0175 - accuracy: 0.9946 - val_loss: 0.8189 - val_accuracy: 0.8594\n",
      "Epoch 89/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0163 - accuracy: 0.9964 - val_loss: 1.0464 - val_accuracy: 0.8281\n",
      "Epoch 90/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0054 - accuracy: 1.0000 - val_loss: 1.1612 - val_accuracy: 0.8750\n",
      "Epoch 91/100\n",
      "556/556 [==============================] - 0s 665us/sample - loss: 0.0232 - accuracy: 0.9928 - val_loss: 1.2047 - val_accuracy: 0.8281\n",
      "Epoch 92/100\n",
      "556/556 [==============================] - 0s 663us/sample - loss: 0.0178 - accuracy: 0.9964 - val_loss: 0.7712 - val_accuracy: 0.8438\n",
      "Epoch 93/100\n",
      "556/556 [==============================] - 0s 664us/sample - loss: 0.0158 - accuracy: 0.9928 - val_loss: 0.8006 - val_accuracy: 0.8438\n",
      "Epoch 94/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0260 - accuracy: 0.9946 - val_loss: 1.1078 - val_accuracy: 0.8281\n",
      "Epoch 95/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0272 - accuracy: 0.9892 - val_loss: 0.6884 - val_accuracy: 0.8438\n",
      "Epoch 96/100\n",
      "556/556 [==============================] - 0s 660us/sample - loss: 0.0150 - accuracy: 0.9946 - val_loss: 1.3108 - val_accuracy: 0.8594\n",
      "Epoch 97/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0136 - accuracy: 0.9982 - val_loss: 1.1392 - val_accuracy: 0.8750\n",
      "Epoch 98/100\n",
      "556/556 [==============================] - 0s 662us/sample - loss: 0.0126 - accuracy: 0.9982 - val_loss: 1.0227 - val_accuracy: 0.8750\n",
      "Epoch 99/100\n",
      "556/556 [==============================] - 0s 659us/sample - loss: 0.0124 - accuracy: 0.9946 - val_loss: 0.9002 - val_accuracy: 0.8594\n",
      "Epoch 100/100\n",
      "556/556 [==============================] - 0s 653us/sample - loss: 0.0084 - accuracy: 0.9946 - val_loss: 0.8061 - val_accuracy: 0.8594\n",
      "accuracy for model 3 is 85.9375\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_18 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 557 samples, validate on 63 samples\n",
      "Epoch 1/100\n",
      "557/557 [==============================] - 1s 2ms/sample - loss: 1.0956 - accuracy: 0.5996 - val_loss: 1.9749 - val_accuracy: 0.8413\n",
      "Epoch 2/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.7103 - accuracy: 0.7971 - val_loss: 1.0736 - val_accuracy: 0.8730\n",
      "Epoch 3/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.6113 - accuracy: 0.8276 - val_loss: 1.0308 - val_accuracy: 0.8095\n",
      "Epoch 4/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.5555 - accuracy: 0.8348 - val_loss: 0.6659 - val_accuracy: 0.8730\n",
      "Epoch 5/100\n",
      "557/557 [==============================] - 0s 761us/sample - loss: 0.5208 - accuracy: 0.8528 - val_loss: 0.6720 - val_accuracy: 0.8730\n",
      "Epoch 6/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.4454 - accuracy: 0.8725 - val_loss: 0.4666 - val_accuracy: 0.8730\n",
      "Epoch 7/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.4562 - accuracy: 0.8474 - val_loss: 0.4227 - val_accuracy: 0.8730\n",
      "Epoch 8/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.4230 - accuracy: 0.8707 - val_loss: 0.3941 - val_accuracy: 0.8730\n",
      "Epoch 9/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.3934 - accuracy: 0.8707 - val_loss: 0.4036 - val_accuracy: 0.8571\n",
      "Epoch 10/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.3775 - accuracy: 0.8725 - val_loss: 0.4005 - val_accuracy: 0.8730\n",
      "Epoch 11/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.3679 - accuracy: 0.8707 - val_loss: 0.3178 - val_accuracy: 0.8730\n",
      "Epoch 12/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.3554 - accuracy: 0.8779 - val_loss: 0.3753 - val_accuracy: 0.8571\n",
      "Epoch 13/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.3453 - accuracy: 0.8743 - val_loss: 0.3855 - val_accuracy: 0.8571\n",
      "Epoch 14/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.3029 - accuracy: 0.8779 - val_loss: 0.3429 - val_accuracy: 0.8889\n",
      "Epoch 15/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.3048 - accuracy: 0.8851 - val_loss: 0.3605 - val_accuracy: 0.8571\n",
      "Epoch 16/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.2973 - accuracy: 0.8887 - val_loss: 0.2987 - val_accuracy: 0.8889\n",
      "Epoch 17/100\n",
      "557/557 [==============================] - 0s 670us/sample - loss: 0.2997 - accuracy: 0.8779 - val_loss: 0.3296 - val_accuracy: 0.8571\n",
      "Epoch 18/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.2561 - accuracy: 0.8869 - val_loss: 0.3830 - val_accuracy: 0.8254\n",
      "Epoch 19/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.2717 - accuracy: 0.8923 - val_loss: 0.3912 - val_accuracy: 0.8413\n",
      "Epoch 20/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.2563 - accuracy: 0.9031 - val_loss: 0.3629 - val_accuracy: 0.8571\n",
      "Epoch 21/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.2244 - accuracy: 0.9210 - val_loss: 0.3522 - val_accuracy: 0.8571\n",
      "Epoch 22/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.2293 - accuracy: 0.9066 - val_loss: 0.3276 - val_accuracy: 0.8730\n",
      "Epoch 23/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.2118 - accuracy: 0.9228 - val_loss: 0.3789 - val_accuracy: 0.8571\n",
      "Epoch 24/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.2330 - accuracy: 0.9174 - val_loss: 0.3810 - val_accuracy: 0.8571\n",
      "Epoch 25/100\n",
      "557/557 [==============================] - 0s 656us/sample - loss: 0.1989 - accuracy: 0.9210 - val_loss: 0.4127 - val_accuracy: 0.8571\n",
      "Epoch 26/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.1651 - accuracy: 0.9443 - val_loss: 0.4190 - val_accuracy: 0.8730\n",
      "Epoch 27/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.1742 - accuracy: 0.9336 - val_loss: 0.4253 - val_accuracy: 0.8730\n",
      "Epoch 28/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.1565 - accuracy: 0.9461 - val_loss: 0.4080 - val_accuracy: 0.8730\n",
      "Epoch 29/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.1513 - accuracy: 0.9354 - val_loss: 0.3906 - val_accuracy: 0.8730\n",
      "Epoch 30/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.1341 - accuracy: 0.9551 - val_loss: 0.3748 - val_accuracy: 0.8254\n",
      "Epoch 31/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.1139 - accuracy: 0.9641 - val_loss: 0.3382 - val_accuracy: 0.8571\n",
      "Epoch 32/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.1134 - accuracy: 0.9551 - val_loss: 0.3596 - val_accuracy: 0.8730\n",
      "Epoch 33/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.1094 - accuracy: 0.9659 - val_loss: 0.3856 - val_accuracy: 0.8889\n",
      "Epoch 34/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.1203 - accuracy: 0.9569 - val_loss: 0.4001 - val_accuracy: 0.9206\n",
      "Epoch 35/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.1043 - accuracy: 0.9623 - val_loss: 0.4275 - val_accuracy: 0.8571\n",
      "Epoch 36/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.0844 - accuracy: 0.9767 - val_loss: 0.3813 - val_accuracy: 0.8889\n",
      "Epoch 37/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0793 - accuracy: 0.9749 - val_loss: 0.4030 - val_accuracy: 0.8571\n",
      "Epoch 38/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0888 - accuracy: 0.9677 - val_loss: 0.3960 - val_accuracy: 0.8571\n",
      "Epoch 39/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0759 - accuracy: 0.9785 - val_loss: 0.3824 - val_accuracy: 0.8889\n",
      "Epoch 40/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0776 - accuracy: 0.9731 - val_loss: 0.5131 - val_accuracy: 0.8254\n",
      "Epoch 41/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0813 - accuracy: 0.9767 - val_loss: 0.5009 - val_accuracy: 0.8571\n",
      "Epoch 42/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0693 - accuracy: 0.9731 - val_loss: 0.3658 - val_accuracy: 0.8889\n",
      "Epoch 43/100\n",
      "557/557 [==============================] - 0s 656us/sample - loss: 0.0741 - accuracy: 0.9713 - val_loss: 0.3990 - val_accuracy: 0.8730\n",
      "Epoch 44/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0837 - accuracy: 0.9659 - val_loss: 0.4332 - val_accuracy: 0.8571\n",
      "Epoch 45/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0491 - accuracy: 0.9856 - val_loss: 0.5352 - val_accuracy: 0.8413\n",
      "Epoch 46/100\n",
      "557/557 [==============================] - 0s 665us/sample - loss: 0.0507 - accuracy: 0.9856 - val_loss: 0.5075 - val_accuracy: 0.8571\n",
      "Epoch 47/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0871 - accuracy: 0.9677 - val_loss: 0.4333 - val_accuracy: 0.8889\n",
      "Epoch 48/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0926 - accuracy: 0.9713 - val_loss: 0.4521 - val_accuracy: 0.8730\n",
      "Epoch 49/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0608 - accuracy: 0.9749 - val_loss: 0.5173 - val_accuracy: 0.8254\n",
      "Epoch 50/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.0486 - accuracy: 0.9874 - val_loss: 0.4298 - val_accuracy: 0.8730\n",
      "Epoch 51/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0391 - accuracy: 0.9892 - val_loss: 0.5282 - val_accuracy: 0.8413\n",
      "Epoch 52/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0778 - accuracy: 0.9731 - val_loss: 0.3925 - val_accuracy: 0.8413\n",
      "Epoch 53/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0672 - accuracy: 0.9785 - val_loss: 0.3756 - val_accuracy: 0.9048\n",
      "Epoch 54/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0621 - accuracy: 0.9820 - val_loss: 0.4232 - val_accuracy: 0.8413\n",
      "Epoch 55/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.0475 - accuracy: 0.9856 - val_loss: 0.4280 - val_accuracy: 0.8730\n",
      "Epoch 56/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0567 - accuracy: 0.9820 - val_loss: 0.4413 - val_accuracy: 0.8730\n",
      "Epoch 57/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.0478 - accuracy: 0.9838 - val_loss: 0.4449 - val_accuracy: 0.9206\n",
      "Epoch 58/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0313 - accuracy: 0.9928 - val_loss: 0.5218 - val_accuracy: 0.8730\n",
      "Epoch 59/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0474 - accuracy: 0.9856 - val_loss: 0.5341 - val_accuracy: 0.8730\n",
      "Epoch 60/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0425 - accuracy: 0.9874 - val_loss: 0.5181 - val_accuracy: 0.8730\n",
      "Epoch 61/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0353 - accuracy: 0.9874 - val_loss: 0.4582 - val_accuracy: 0.8413\n",
      "Epoch 62/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0298 - accuracy: 0.9928 - val_loss: 0.4604 - val_accuracy: 0.8413\n",
      "Epoch 63/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0310 - accuracy: 0.9910 - val_loss: 0.4386 - val_accuracy: 0.8730\n",
      "Epoch 64/100\n",
      "557/557 [==============================] - 0s 659us/sample - loss: 0.0142 - accuracy: 0.9982 - val_loss: 0.4881 - val_accuracy: 0.8571\n",
      "Epoch 65/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0216 - accuracy: 0.9964 - val_loss: 0.4480 - val_accuracy: 0.8730\n",
      "Epoch 66/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0202 - accuracy: 0.9928 - val_loss: 0.5053 - val_accuracy: 0.8571\n",
      "Epoch 67/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0265 - accuracy: 0.9910 - val_loss: 0.4352 - val_accuracy: 0.8889\n",
      "Epoch 68/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.0490 - accuracy: 0.9820 - val_loss: 0.4817 - val_accuracy: 0.8571\n",
      "Epoch 69/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.0203 - accuracy: 0.9964 - val_loss: 0.5992 - val_accuracy: 0.8730\n",
      "Epoch 70/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0262 - accuracy: 0.9910 - val_loss: 0.4181 - val_accuracy: 0.9048\n",
      "Epoch 71/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0170 - accuracy: 0.9964 - val_loss: 0.6368 - val_accuracy: 0.8095\n",
      "Epoch 72/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0374 - accuracy: 0.9910 - val_loss: 0.5222 - val_accuracy: 0.8571\n",
      "Epoch 73/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0191 - accuracy: 0.9910 - val_loss: 0.5428 - val_accuracy: 0.8730\n",
      "Epoch 74/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0260 - accuracy: 0.9910 - val_loss: 0.9183 - val_accuracy: 0.6825\n",
      "Epoch 75/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0267 - accuracy: 0.9928 - val_loss: 0.5132 - val_accuracy: 0.8730\n",
      "Epoch 76/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0238 - accuracy: 0.9928 - val_loss: 0.4702 - val_accuracy: 0.8571\n",
      "Epoch 77/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0320 - accuracy: 0.9910 - val_loss: 0.5040 - val_accuracy: 0.8730\n",
      "Epoch 78/100\n",
      "557/557 [==============================] - 0s 656us/sample - loss: 0.0277 - accuracy: 0.9910 - val_loss: 0.5503 - val_accuracy: 0.8254\n",
      "Epoch 79/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0214 - accuracy: 0.9928 - val_loss: 0.5876 - val_accuracy: 0.8730\n",
      "Epoch 80/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0153 - accuracy: 0.9982 - val_loss: 0.5484 - val_accuracy: 0.8571\n",
      "Epoch 81/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0320 - accuracy: 0.9928 - val_loss: 0.5567 - val_accuracy: 0.9206\n",
      "Epoch 82/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0223 - accuracy: 0.9964 - val_loss: 0.5880 - val_accuracy: 0.8571\n",
      "Epoch 83/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0196 - accuracy: 0.9946 - val_loss: 0.4934 - val_accuracy: 0.9048\n",
      "Epoch 84/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0126 - accuracy: 0.9964 - val_loss: 0.4315 - val_accuracy: 0.8571\n",
      "Epoch 85/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0139 - accuracy: 0.9964 - val_loss: 0.4728 - val_accuracy: 0.9048\n",
      "Epoch 86/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0090 - accuracy: 0.9982 - val_loss: 0.6180 - val_accuracy: 0.8730\n",
      "Epoch 87/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0128 - accuracy: 0.9982 - val_loss: 0.4815 - val_accuracy: 0.8730\n",
      "Epoch 88/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0082 - accuracy: 0.9982 - val_loss: 0.4730 - val_accuracy: 0.9048\n",
      "Epoch 89/100\n",
      "557/557 [==============================] - 0s 660us/sample - loss: 0.0127 - accuracy: 0.9946 - val_loss: 0.5396 - val_accuracy: 0.8571\n",
      "Epoch 90/100\n",
      "557/557 [==============================] - 0s 662us/sample - loss: 0.0103 - accuracy: 0.9982 - val_loss: 0.8409 - val_accuracy: 0.7778\n",
      "Epoch 91/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0319 - accuracy: 0.9856 - val_loss: 0.6191 - val_accuracy: 0.9048\n",
      "Epoch 92/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0512 - accuracy: 0.9785 - val_loss: 0.7815 - val_accuracy: 0.8889\n",
      "Epoch 93/100\n",
      "557/557 [==============================] - 0s 664us/sample - loss: 0.0418 - accuracy: 0.9856 - val_loss: 0.6567 - val_accuracy: 0.8413\n",
      "Epoch 94/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0115 - accuracy: 0.9964 - val_loss: 0.6459 - val_accuracy: 0.8571\n",
      "Epoch 95/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0210 - accuracy: 0.9946 - val_loss: 0.7110 - val_accuracy: 0.8254\n",
      "Epoch 96/100\n",
      "557/557 [==============================] - 0s 663us/sample - loss: 0.0155 - accuracy: 0.9946 - val_loss: 0.6423 - val_accuracy: 0.8413\n",
      "Epoch 97/100\n",
      "557/557 [==============================] - 0s 657us/sample - loss: 0.0154 - accuracy: 0.9946 - val_loss: 0.5895 - val_accuracy: 0.8571\n",
      "Epoch 98/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0072 - accuracy: 1.0000 - val_loss: 0.4500 - val_accuracy: 0.8730\n",
      "Epoch 99/100\n",
      "557/557 [==============================] - 0s 661us/sample - loss: 0.0220 - accuracy: 0.9982 - val_loss: 0.5085 - val_accuracy: 0.8889\n",
      "Epoch 100/100\n",
      "557/557 [==============================] - 0s 656us/sample - loss: 0.0124 - accuracy: 0.9928 - val_loss: 0.5226 - val_accuracy: 0.8730\n",
      "accuracy for model 4 is 87.30158805847168\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_24 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_23 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_24 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 558 samples, validate on 62 samples\n",
      "Epoch 1/100\n",
      "558/558 [==============================] - 1s 2ms/sample - loss: 0.9226 - accuracy: 0.6685 - val_loss: 0.9993 - val_accuracy: 0.7419\n",
      "Epoch 2/100\n",
      "558/558 [==============================] - 0s 666us/sample - loss: 0.5624 - accuracy: 0.8423 - val_loss: 0.6574 - val_accuracy: 0.8710\n",
      "Epoch 3/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.4834 - accuracy: 0.8602 - val_loss: 0.4731 - val_accuracy: 0.8871\n",
      "Epoch 4/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.4505 - accuracy: 0.8548 - val_loss: 0.4613 - val_accuracy: 0.8871\n",
      "Epoch 5/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.4127 - accuracy: 0.8638 - val_loss: 0.3613 - val_accuracy: 0.9032\n",
      "Epoch 6/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.3831 - accuracy: 0.8710 - val_loss: 0.3484 - val_accuracy: 0.8871\n",
      "Epoch 7/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.3664 - accuracy: 0.8781 - val_loss: 0.3822 - val_accuracy: 0.9032\n",
      "Epoch 8/100\n",
      "558/558 [==============================] - 0s 676us/sample - loss: 0.3549 - accuracy: 0.8853 - val_loss: 0.3755 - val_accuracy: 0.8871\n",
      "Epoch 9/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.3334 - accuracy: 0.8817 - val_loss: 0.4597 - val_accuracy: 0.8710\n",
      "Epoch 10/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.3069 - accuracy: 0.9050 - val_loss: 0.3402 - val_accuracy: 0.8871\n",
      "Epoch 11/100\n",
      "558/558 [==============================] - 0s 664us/sample - loss: 0.2966 - accuracy: 0.8961 - val_loss: 0.3674 - val_accuracy: 0.8710\n",
      "Epoch 12/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2876 - accuracy: 0.8943 - val_loss: 0.4776 - val_accuracy: 0.8710\n",
      "Epoch 13/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2855 - accuracy: 0.9122 - val_loss: 0.3733 - val_accuracy: 0.8871\n",
      "Epoch 14/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2541 - accuracy: 0.9050 - val_loss: 0.5012 - val_accuracy: 0.8548\n",
      "Epoch 15/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2622 - accuracy: 0.8961 - val_loss: 0.7366 - val_accuracy: 0.7903\n",
      "Epoch 16/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2205 - accuracy: 0.9158 - val_loss: 0.4627 - val_accuracy: 0.8871\n",
      "Epoch 17/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.2059 - accuracy: 0.9229 - val_loss: 0.4084 - val_accuracy: 0.8871\n",
      "Epoch 18/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.1790 - accuracy: 0.9337 - val_loss: 0.4641 - val_accuracy: 0.8548\n",
      "Epoch 19/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.1837 - accuracy: 0.9373 - val_loss: 0.5117 - val_accuracy: 0.8710\n",
      "Epoch 20/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.1768 - accuracy: 0.9337 - val_loss: 0.6293 - val_accuracy: 0.8387\n",
      "Epoch 21/100\n",
      "558/558 [==============================] - 0s 687us/sample - loss: 0.1760 - accuracy: 0.9391 - val_loss: 0.4303 - val_accuracy: 0.8871\n",
      "Epoch 22/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.1488 - accuracy: 0.9552 - val_loss: 0.4981 - val_accuracy: 0.8710\n",
      "Epoch 23/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.1413 - accuracy: 0.9516 - val_loss: 0.4816 - val_accuracy: 0.8710\n",
      "Epoch 24/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.1318 - accuracy: 0.9642 - val_loss: 0.5295 - val_accuracy: 0.8548\n",
      "Epoch 25/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0969 - accuracy: 0.9713 - val_loss: 0.5184 - val_accuracy: 0.8548\n",
      "Epoch 26/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0801 - accuracy: 0.9875 - val_loss: 0.5988 - val_accuracy: 0.8548\n",
      "Epoch 27/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0792 - accuracy: 0.9803 - val_loss: 0.5505 - val_accuracy: 0.8871\n",
      "Epoch 28/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0778 - accuracy: 0.9749 - val_loss: 0.5674 - val_accuracy: 0.7903\n",
      "Epoch 29/100\n",
      "558/558 [==============================] - 0s 686us/sample - loss: 0.0727 - accuracy: 0.9767 - val_loss: 0.6941 - val_accuracy: 0.8710\n",
      "Epoch 30/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0699 - accuracy: 0.9839 - val_loss: 0.5722 - val_accuracy: 0.8871\n",
      "Epoch 31/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0869 - accuracy: 0.9642 - val_loss: 0.8428 - val_accuracy: 0.8710\n",
      "Epoch 32/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0625 - accuracy: 0.9839 - val_loss: 0.6715 - val_accuracy: 0.8710\n",
      "Epoch 33/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0608 - accuracy: 0.9821 - val_loss: 0.5993 - val_accuracy: 0.9032\n",
      "Epoch 34/100\n",
      "558/558 [==============================] - 0s 665us/sample - loss: 0.0703 - accuracy: 0.9767 - val_loss: 0.7133 - val_accuracy: 0.8871\n",
      "Epoch 35/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0486 - accuracy: 0.9875 - val_loss: 0.5986 - val_accuracy: 0.8548\n",
      "Epoch 36/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0389 - accuracy: 0.9875 - val_loss: 0.8297 - val_accuracy: 0.8226\n",
      "Epoch 37/100\n",
      "558/558 [==============================] - 0s 693us/sample - loss: 0.0293 - accuracy: 0.9982 - val_loss: 0.7203 - val_accuracy: 0.8710\n",
      "Epoch 38/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0303 - accuracy: 0.9892 - val_loss: 0.7151 - val_accuracy: 0.8871\n",
      "Epoch 39/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0258 - accuracy: 0.9964 - val_loss: 0.7984 - val_accuracy: 0.8710\n",
      "Epoch 40/100\n",
      "558/558 [==============================] - 0s 679us/sample - loss: 0.0427 - accuracy: 0.9892 - val_loss: 1.1138 - val_accuracy: 0.6774\n",
      "Epoch 41/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0674 - accuracy: 0.9803 - val_loss: 0.7428 - val_accuracy: 0.8710\n",
      "Epoch 42/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0320 - accuracy: 0.9928 - val_loss: 0.8796 - val_accuracy: 0.8710\n",
      "Epoch 43/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0318 - accuracy: 0.9875 - val_loss: 0.8919 - val_accuracy: 0.7742\n",
      "Epoch 44/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0345 - accuracy: 0.9910 - val_loss: 0.7310 - val_accuracy: 0.8871\n",
      "Epoch 45/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0746 - accuracy: 0.9821 - val_loss: 1.4017 - val_accuracy: 0.8871\n",
      "Epoch 46/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0677 - accuracy: 0.9767 - val_loss: 0.6575 - val_accuracy: 0.8710\n",
      "Epoch 47/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0552 - accuracy: 0.9785 - val_loss: 0.6336 - val_accuracy: 0.8710\n",
      "Epoch 48/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0238 - accuracy: 0.9910 - val_loss: 0.8115 - val_accuracy: 0.8871\n",
      "Epoch 49/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0291 - accuracy: 0.9928 - val_loss: 0.7593 - val_accuracy: 0.8871\n",
      "Epoch 50/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0519 - accuracy: 0.9857 - val_loss: 0.7597 - val_accuracy: 0.9032\n",
      "Epoch 51/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0171 - accuracy: 0.9946 - val_loss: 0.8458 - val_accuracy: 0.8871\n",
      "Epoch 52/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0183 - accuracy: 0.9946 - val_loss: 0.8628 - val_accuracy: 0.8871\n",
      "Epoch 53/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0263 - accuracy: 0.9946 - val_loss: 0.8857 - val_accuracy: 0.7903\n",
      "Epoch 54/100\n",
      "558/558 [==============================] - 0s 693us/sample - loss: 0.0251 - accuracy: 0.9946 - val_loss: 0.9764 - val_accuracy: 0.8871\n",
      "Epoch 55/100\n",
      "558/558 [==============================] - 0s 665us/sample - loss: 0.0183 - accuracy: 0.9964 - val_loss: 0.7785 - val_accuracy: 0.8871\n",
      "Epoch 56/100\n",
      "558/558 [==============================] - 0s 693us/sample - loss: 0.0090 - accuracy: 1.0000 - val_loss: 0.7606 - val_accuracy: 0.8387\n",
      "Epoch 57/100\n",
      "558/558 [==============================] - 0s 686us/sample - loss: 0.0083 - accuracy: 1.0000 - val_loss: 0.8061 - val_accuracy: 0.8710\n",
      "Epoch 58/100\n",
      "558/558 [==============================] - 0s 684us/sample - loss: 0.0236 - accuracy: 0.9946 - val_loss: 0.7733 - val_accuracy: 0.8548\n",
      "Epoch 59/100\n",
      "558/558 [==============================] - 0s 696us/sample - loss: 0.0140 - accuracy: 0.9964 - val_loss: 0.7829 - val_accuracy: 0.8387\n",
      "Epoch 60/100\n",
      "558/558 [==============================] - 0s 691us/sample - loss: 0.0086 - accuracy: 0.9982 - val_loss: 0.9177 - val_accuracy: 0.8871\n",
      "Epoch 61/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0099 - accuracy: 0.9964 - val_loss: 0.8110 - val_accuracy: 0.8710\n",
      "Epoch 62/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0126 - accuracy: 0.9982 - val_loss: 0.8425 - val_accuracy: 0.8548\n",
      "Epoch 63/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0131 - accuracy: 0.9982 - val_loss: 0.9149 - val_accuracy: 0.8710\n",
      "Epoch 64/100\n",
      "558/558 [==============================] - 0s 692us/sample - loss: 0.0104 - accuracy: 0.9982 - val_loss: 0.8175 - val_accuracy: 0.8548\n",
      "Epoch 65/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0088 - accuracy: 0.9982 - val_loss: 0.8539 - val_accuracy: 0.8387\n",
      "Epoch 66/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0118 - accuracy: 0.9964 - val_loss: 0.8698 - val_accuracy: 0.9032\n",
      "Epoch 67/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0180 - accuracy: 0.9982 - val_loss: 0.8251 - val_accuracy: 0.8548\n",
      "Epoch 68/100\n",
      "558/558 [==============================] - 0s 680us/sample - loss: 0.0075 - accuracy: 0.9982 - val_loss: 0.7833 - val_accuracy: 0.9032\n",
      "Epoch 69/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0068 - accuracy: 1.0000 - val_loss: 0.8425 - val_accuracy: 0.8871\n",
      "Epoch 70/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0204 - accuracy: 0.9946 - val_loss: 0.9000 - val_accuracy: 0.8548\n",
      "Epoch 71/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0050 - accuracy: 1.0000 - val_loss: 0.9203 - val_accuracy: 0.8710\n",
      "Epoch 72/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0103 - accuracy: 0.9964 - val_loss: 0.8673 - val_accuracy: 0.8710\n",
      "Epoch 73/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0040 - accuracy: 1.0000 - val_loss: 0.9345 - val_accuracy: 0.8387\n",
      "Epoch 74/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0067 - accuracy: 0.9982 - val_loss: 0.9033 - val_accuracy: 0.8548\n",
      "Epoch 75/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0027 - accuracy: 1.0000 - val_loss: 0.8869 - val_accuracy: 0.8548\n",
      "Epoch 76/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0146 - accuracy: 0.9964 - val_loss: 0.9655 - val_accuracy: 0.8226\n",
      "Epoch 77/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0568 - accuracy: 0.9821 - val_loss: 1.1536 - val_accuracy: 0.6774\n",
      "Epoch 78/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0317 - accuracy: 0.9892 - val_loss: 0.8766 - val_accuracy: 0.7903\n",
      "Epoch 79/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0444 - accuracy: 0.9875 - val_loss: 0.6780 - val_accuracy: 0.8548\n",
      "Epoch 80/100\n",
      "558/558 [==============================] - 0s 670us/sample - loss: 0.0215 - accuracy: 0.9928 - val_loss: 0.7904 - val_accuracy: 0.8710\n",
      "Epoch 81/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0303 - accuracy: 0.9892 - val_loss: 0.8639 - val_accuracy: 0.8710\n",
      "Epoch 82/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0139 - accuracy: 0.9946 - val_loss: 0.8621 - val_accuracy: 0.8710\n",
      "Epoch 83/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0092 - accuracy: 0.9982 - val_loss: 0.8225 - val_accuracy: 0.8710\n",
      "Epoch 84/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0157 - accuracy: 0.9946 - val_loss: 0.8393 - val_accuracy: 0.8548\n",
      "Epoch 85/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0165 - accuracy: 0.9946 - val_loss: 1.0069 - val_accuracy: 0.8226\n",
      "Epoch 86/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0255 - accuracy: 0.9875 - val_loss: 0.8617 - val_accuracy: 0.8871\n",
      "Epoch 87/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0139 - accuracy: 0.9946 - val_loss: 0.8156 - val_accuracy: 0.8871\n",
      "Epoch 88/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0187 - accuracy: 0.9928 - val_loss: 0.7871 - val_accuracy: 0.8548\n",
      "Epoch 89/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0149 - accuracy: 0.9946 - val_loss: 0.8301 - val_accuracy: 0.8710\n",
      "Epoch 90/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0092 - accuracy: 0.9964 - val_loss: 0.9570 - val_accuracy: 0.8710\n",
      "Epoch 91/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0054 - accuracy: 0.9982 - val_loss: 0.9321 - val_accuracy: 0.8871\n",
      "Epoch 92/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0034 - accuracy: 1.0000 - val_loss: 0.9488 - val_accuracy: 0.8871\n",
      "Epoch 93/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0087 - accuracy: 0.9982 - val_loss: 0.9378 - val_accuracy: 0.8871\n",
      "Epoch 94/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0095 - accuracy: 0.9982 - val_loss: 0.9815 - val_accuracy: 0.8710\n",
      "Epoch 95/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0051 - accuracy: 0.9982 - val_loss: 0.9417 - val_accuracy: 0.8710\n",
      "Epoch 96/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0072 - accuracy: 0.9982 - val_loss: 0.9251 - val_accuracy: 0.8710\n",
      "Epoch 97/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0046 - accuracy: 1.0000 - val_loss: 0.8876 - val_accuracy: 0.8710\n",
      "Epoch 98/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0063 - accuracy: 0.9982 - val_loss: 0.9407 - val_accuracy: 0.8710\n",
      "Epoch 99/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.0106 - accuracy: 0.9964 - val_loss: 0.9093 - val_accuracy: 0.8871\n",
      "Epoch 100/100\n",
      "558/558 [==============================] - 0s 649us/sample - loss: 0.0223 - accuracy: 0.9964 - val_loss: 0.9328 - val_accuracy: 0.8710\n",
      "accuracy for model 5 is 87.09677457809448\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_30 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_25 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_26 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_27 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_17 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_28 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_29 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 558 samples, validate on 62 samples\n",
      "Epoch 1/100\n",
      "558/558 [==============================] - 1s 2ms/sample - loss: 0.9427 - accuracy: 0.6828 - val_loss: 2.0824 - val_accuracy: 0.6935\n",
      "Epoch 2/100\n",
      "558/558 [==============================] - 0s 666us/sample - loss: 0.6294 - accuracy: 0.8244 - val_loss: 0.5931 - val_accuracy: 0.8871\n",
      "Epoch 3/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.5591 - accuracy: 0.8244 - val_loss: 0.6337 - val_accuracy: 0.8871\n",
      "Epoch 4/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.5091 - accuracy: 0.8477 - val_loss: 0.6363 - val_accuracy: 0.8871\n",
      "Epoch 5/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.4543 - accuracy: 0.8638 - val_loss: 0.5157 - val_accuracy: 0.9032\n",
      "Epoch 6/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.4265 - accuracy: 0.8692 - val_loss: 0.3931 - val_accuracy: 0.8871\n",
      "Epoch 7/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.4026 - accuracy: 0.8674 - val_loss: 0.5402 - val_accuracy: 0.7903\n",
      "Epoch 8/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.3815 - accuracy: 0.8835 - val_loss: 0.4568 - val_accuracy: 0.8548\n",
      "Epoch 9/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.3694 - accuracy: 0.8763 - val_loss: 0.4871 - val_accuracy: 0.9032\n",
      "Epoch 10/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.3404 - accuracy: 0.8871 - val_loss: 0.3974 - val_accuracy: 0.8871\n",
      "Epoch 11/100\n",
      "558/558 [==============================] - 0s 654us/sample - loss: 0.3417 - accuracy: 0.8781 - val_loss: 0.4341 - val_accuracy: 0.9032\n",
      "Epoch 12/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.3138 - accuracy: 0.8925 - val_loss: 0.3402 - val_accuracy: 0.9032\n",
      "Epoch 13/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.3195 - accuracy: 0.8746 - val_loss: 0.3276 - val_accuracy: 0.9194\n",
      "Epoch 14/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2736 - accuracy: 0.8961 - val_loss: 0.4127 - val_accuracy: 0.8871\n",
      "Epoch 15/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.2608 - accuracy: 0.9068 - val_loss: 0.5046 - val_accuracy: 0.8548\n",
      "Epoch 16/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2393 - accuracy: 0.8996 - val_loss: 0.3691 - val_accuracy: 0.9032\n",
      "Epoch 17/100\n",
      "558/558 [==============================] - 0s 663us/sample - loss: 0.2456 - accuracy: 0.9068 - val_loss: 0.5088 - val_accuracy: 0.8710\n",
      "Epoch 18/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.2281 - accuracy: 0.9086 - val_loss: 0.5191 - val_accuracy: 0.8226\n",
      "Epoch 19/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.2135 - accuracy: 0.9247 - val_loss: 0.7052 - val_accuracy: 0.8065\n",
      "Epoch 20/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.2108 - accuracy: 0.9158 - val_loss: 0.3130 - val_accuracy: 0.9032\n",
      "Epoch 21/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.1635 - accuracy: 0.9337 - val_loss: 0.5414 - val_accuracy: 0.9032\n",
      "Epoch 22/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.1467 - accuracy: 0.9534 - val_loss: 0.4986 - val_accuracy: 0.9032\n",
      "Epoch 23/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.1611 - accuracy: 0.9355 - val_loss: 0.4670 - val_accuracy: 0.7742\n",
      "Epoch 24/100\n",
      "558/558 [==============================] - 0s 682us/sample - loss: 0.1232 - accuracy: 0.9606 - val_loss: 0.5583 - val_accuracy: 0.8710\n",
      "Epoch 25/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0992 - accuracy: 0.9588 - val_loss: 0.4357 - val_accuracy: 0.8710\n",
      "Epoch 26/100\n",
      "558/558 [==============================] - 0s 654us/sample - loss: 0.1034 - accuracy: 0.9659 - val_loss: 0.7064 - val_accuracy: 0.9032\n",
      "Epoch 27/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0816 - accuracy: 0.9749 - val_loss: 0.5301 - val_accuracy: 0.9032\n",
      "Epoch 28/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0867 - accuracy: 0.9749 - val_loss: 0.5153 - val_accuracy: 0.9032\n",
      "Epoch 29/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0661 - accuracy: 0.9839 - val_loss: 0.4717 - val_accuracy: 0.9032\n",
      "Epoch 30/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0643 - accuracy: 0.9785 - val_loss: 0.4143 - val_accuracy: 0.8387\n",
      "Epoch 31/100\n",
      "558/558 [==============================] - 0s 683us/sample - loss: 0.0628 - accuracy: 0.9839 - val_loss: 0.5705 - val_accuracy: 0.8871\n",
      "Epoch 32/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0388 - accuracy: 0.9892 - val_loss: 0.5720 - val_accuracy: 0.9032\n",
      "Epoch 33/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0839 - accuracy: 0.9749 - val_loss: 0.7840 - val_accuracy: 0.8871\n",
      "Epoch 34/100\n",
      "558/558 [==============================] - 0s 686us/sample - loss: 0.0636 - accuracy: 0.9803 - val_loss: 0.5582 - val_accuracy: 0.9032\n",
      "Epoch 35/100\n",
      "558/558 [==============================] - 0s 681us/sample - loss: 0.0446 - accuracy: 0.9892 - val_loss: 0.5648 - val_accuracy: 0.8548\n",
      "Epoch 36/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0531 - accuracy: 0.9821 - val_loss: 0.6631 - val_accuracy: 0.9032\n",
      "Epoch 37/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0278 - accuracy: 0.9946 - val_loss: 0.6418 - val_accuracy: 0.8710\n",
      "Epoch 38/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0462 - accuracy: 0.9803 - val_loss: 0.5680 - val_accuracy: 0.9032\n",
      "Epoch 39/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0521 - accuracy: 0.9839 - val_loss: 0.5628 - val_accuracy: 0.9032\n",
      "Epoch 40/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0350 - accuracy: 0.9928 - val_loss: 0.6488 - val_accuracy: 0.9032\n",
      "Epoch 41/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0345 - accuracy: 0.9892 - val_loss: 0.6380 - val_accuracy: 0.8548\n",
      "Epoch 42/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0411 - accuracy: 0.9839 - val_loss: 0.4664 - val_accuracy: 0.8871\n",
      "Epoch 43/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0423 - accuracy: 0.9946 - val_loss: 0.6668 - val_accuracy: 0.9032\n",
      "Epoch 44/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0560 - accuracy: 0.9785 - val_loss: 0.6819 - val_accuracy: 0.9032\n",
      "Epoch 45/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0268 - accuracy: 0.9892 - val_loss: 0.7534 - val_accuracy: 0.9032\n",
      "Epoch 46/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0189 - accuracy: 0.9964 - val_loss: 0.7569 - val_accuracy: 0.9032\n",
      "Epoch 47/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0099 - accuracy: 1.0000 - val_loss: 0.6648 - val_accuracy: 0.9032\n",
      "Epoch 48/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0263 - accuracy: 0.9910 - val_loss: 0.6859 - val_accuracy: 0.8226\n",
      "Epoch 49/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0186 - accuracy: 0.9946 - val_loss: 0.7745 - val_accuracy: 0.9032\n",
      "Epoch 50/100\n",
      "558/558 [==============================] - 0s 665us/sample - loss: 0.0194 - accuracy: 0.9946 - val_loss: 0.7753 - val_accuracy: 0.8871\n",
      "Epoch 51/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0183 - accuracy: 0.9946 - val_loss: 0.7352 - val_accuracy: 0.8387\n",
      "Epoch 52/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0241 - accuracy: 0.9928 - val_loss: 0.7633 - val_accuracy: 0.8871\n",
      "Epoch 53/100\n",
      "558/558 [==============================] - 0s 692us/sample - loss: 0.0141 - accuracy: 0.9964 - val_loss: 0.7382 - val_accuracy: 0.8387\n",
      "Epoch 54/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0262 - accuracy: 0.9892 - val_loss: 1.6607 - val_accuracy: 0.6129\n",
      "Epoch 55/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0676 - accuracy: 0.9821 - val_loss: 0.7003 - val_accuracy: 0.9032\n",
      "Epoch 56/100\n",
      "558/558 [==============================] - 0s 684us/sample - loss: 0.0199 - accuracy: 0.9928 - val_loss: 0.5953 - val_accuracy: 0.8548\n",
      "Epoch 57/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0199 - accuracy: 0.9928 - val_loss: 0.8912 - val_accuracy: 0.9032\n",
      "Epoch 58/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0147 - accuracy: 0.9964 - val_loss: 0.7999 - val_accuracy: 0.8710\n",
      "Epoch 59/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0158 - accuracy: 0.9964 - val_loss: 0.7731 - val_accuracy: 0.8710\n",
      "Epoch 60/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0192 - accuracy: 0.9928 - val_loss: 0.7462 - val_accuracy: 0.9032\n",
      "Epoch 61/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0291 - accuracy: 0.9910 - val_loss: 0.6897 - val_accuracy: 0.9032\n",
      "Epoch 62/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0186 - accuracy: 0.9964 - val_loss: 0.7620 - val_accuracy: 0.8548\n",
      "Epoch 63/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0135 - accuracy: 0.9964 - val_loss: 0.7334 - val_accuracy: 0.8871\n",
      "Epoch 64/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0129 - accuracy: 0.9946 - val_loss: 0.6450 - val_accuracy: 0.8871\n",
      "Epoch 65/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0118 - accuracy: 0.9982 - val_loss: 0.7362 - val_accuracy: 0.9032\n",
      "Epoch 66/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0143 - accuracy: 0.9964 - val_loss: 0.6678 - val_accuracy: 0.8871\n",
      "Epoch 67/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0149 - accuracy: 0.9946 - val_loss: 0.5873 - val_accuracy: 0.9032\n",
      "Epoch 68/100\n",
      "558/558 [==============================] - 0s 654us/sample - loss: 0.0253 - accuracy: 0.9946 - val_loss: 0.7296 - val_accuracy: 0.9032\n",
      "Epoch 69/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0270 - accuracy: 0.9910 - val_loss: 0.7888 - val_accuracy: 0.9032\n",
      "Epoch 70/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0357 - accuracy: 0.9857 - val_loss: 0.6458 - val_accuracy: 0.8065\n",
      "Epoch 71/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0126 - accuracy: 0.9982 - val_loss: 0.9268 - val_accuracy: 0.8548\n",
      "Epoch 72/100\n",
      "558/558 [==============================] - 0s 661us/sample - loss: 0.0149 - accuracy: 0.9946 - val_loss: 0.6347 - val_accuracy: 0.8226\n",
      "Epoch 73/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0097 - accuracy: 0.9964 - val_loss: 0.8657 - val_accuracy: 0.9032\n",
      "Epoch 74/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0115 - accuracy: 0.9964 - val_loss: 0.9015 - val_accuracy: 0.9032\n",
      "Epoch 75/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0347 - accuracy: 0.9892 - val_loss: 0.8236 - val_accuracy: 0.9032\n",
      "Epoch 76/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0300 - accuracy: 0.9928 - val_loss: 0.6273 - val_accuracy: 0.8871\n",
      "Epoch 77/100\n",
      "558/558 [==============================] - 0s 673us/sample - loss: 0.0180 - accuracy: 0.9946 - val_loss: 0.7945 - val_accuracy: 0.8710\n",
      "Epoch 78/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0193 - accuracy: 0.9928 - val_loss: 1.1261 - val_accuracy: 0.9032\n",
      "Epoch 79/100\n",
      "558/558 [==============================] - 0s 662us/sample - loss: 0.0271 - accuracy: 0.9910 - val_loss: 0.5370 - val_accuracy: 0.9032\n",
      "Epoch 80/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0144 - accuracy: 0.9946 - val_loss: 0.6970 - val_accuracy: 0.9032\n",
      "Epoch 81/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0229 - accuracy: 0.9928 - val_loss: 0.7518 - val_accuracy: 0.9032\n",
      "Epoch 82/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0186 - accuracy: 0.9946 - val_loss: 0.8325 - val_accuracy: 0.9032\n",
      "Epoch 83/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0192 - accuracy: 0.9946 - val_loss: 0.9863 - val_accuracy: 0.9032\n",
      "Epoch 84/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0221 - accuracy: 0.9946 - val_loss: 0.9123 - val_accuracy: 0.9032\n",
      "Epoch 85/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0137 - accuracy: 0.9964 - val_loss: 0.6120 - val_accuracy: 0.8710\n",
      "Epoch 86/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0136 - accuracy: 0.9964 - val_loss: 0.7610 - val_accuracy: 0.8548\n",
      "Epoch 87/100\n",
      "558/558 [==============================] - 0s 655us/sample - loss: 0.0060 - accuracy: 0.9982 - val_loss: 0.6755 - val_accuracy: 0.8710\n",
      "Epoch 88/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0127 - accuracy: 0.9964 - val_loss: 0.7890 - val_accuracy: 0.8871\n",
      "Epoch 89/100\n",
      "558/558 [==============================] - 0s 657us/sample - loss: 0.0106 - accuracy: 0.9964 - val_loss: 0.7957 - val_accuracy: 0.9032\n",
      "Epoch 90/100\n",
      "558/558 [==============================] - 0s 659us/sample - loss: 0.0110 - accuracy: 0.9964 - val_loss: 0.6635 - val_accuracy: 0.9032\n",
      "Epoch 91/100\n",
      "558/558 [==============================] - 0s 658us/sample - loss: 0.0074 - accuracy: 0.9964 - val_loss: 0.6981 - val_accuracy: 0.9032\n",
      "Epoch 92/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0054 - accuracy: 1.0000 - val_loss: 0.7759 - val_accuracy: 0.9032\n",
      "Epoch 93/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0057 - accuracy: 0.9982 - val_loss: 0.8689 - val_accuracy: 0.9032\n",
      "Epoch 94/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0198 - accuracy: 0.9946 - val_loss: 0.8558 - val_accuracy: 0.9032\n",
      "Epoch 95/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0132 - accuracy: 0.9946 - val_loss: 0.8769 - val_accuracy: 0.9032\n",
      "Epoch 96/100\n",
      "558/558 [==============================] - 0s 667us/sample - loss: 0.0310 - accuracy: 0.9910 - val_loss: 0.7680 - val_accuracy: 0.8871\n",
      "Epoch 97/100\n",
      "558/558 [==============================] - 0s 685us/sample - loss: 0.0081 - accuracy: 0.9982 - val_loss: 0.6016 - val_accuracy: 0.8710\n",
      "Epoch 98/100\n",
      "558/558 [==============================] - 0s 656us/sample - loss: 0.0153 - accuracy: 0.9946 - val_loss: 0.6070 - val_accuracy: 0.8226\n",
      "Epoch 99/100\n",
      "558/558 [==============================] - 0s 660us/sample - loss: 0.0177 - accuracy: 0.9946 - val_loss: 0.7427 - val_accuracy: 0.9032\n",
      "Epoch 100/100\n",
      "558/558 [==============================] - 0s 647us/sample - loss: 0.0121 - accuracy: 0.9982 - val_loss: 0.8002 - val_accuracy: 0.9032\n",
      "accuracy for model 6 is 90.32257795333862\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_36 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_30 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_31 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_32 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_20 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_33 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_40 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_34 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_41 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 559 samples, validate on 61 samples\n",
      "Epoch 1/100\n",
      "559/559 [==============================] - 1s 2ms/sample - loss: 1.2990 - accuracy: 0.4490 - val_loss: 4.3926 - val_accuracy: 0.6721\n",
      "Epoch 2/100\n",
      "559/559 [==============================] - 0s 663us/sample - loss: 0.7779 - accuracy: 0.7513 - val_loss: 3.1350 - val_accuracy: 0.8033\n",
      "Epoch 3/100\n",
      "559/559 [==============================] - 0s 657us/sample - loss: 0.6641 - accuracy: 0.7979 - val_loss: 2.6191 - val_accuracy: 0.8033\n",
      "Epoch 4/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.5876 - accuracy: 0.8265 - val_loss: 1.8395 - val_accuracy: 0.8033\n",
      "Epoch 5/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.5782 - accuracy: 0.8354 - val_loss: 1.3532 - val_accuracy: 0.8197\n",
      "Epoch 6/100\n",
      "559/559 [==============================] - 0s 662us/sample - loss: 0.5289 - accuracy: 0.8283 - val_loss: 1.0087 - val_accuracy: 0.8361\n",
      "Epoch 7/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.5368 - accuracy: 0.8462 - val_loss: 0.7389 - val_accuracy: 0.8689\n",
      "Epoch 8/100\n",
      "559/559 [==============================] - 0s 654us/sample - loss: 0.4988 - accuracy: 0.8426 - val_loss: 0.7011 - val_accuracy: 0.8525\n",
      "Epoch 9/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.4449 - accuracy: 0.8640 - val_loss: 0.5713 - val_accuracy: 0.8689\n",
      "Epoch 10/100\n",
      "559/559 [==============================] - 0s 653us/sample - loss: 0.4648 - accuracy: 0.8533 - val_loss: 0.4469 - val_accuracy: 0.8689\n",
      "Epoch 11/100\n",
      "559/559 [==============================] - 0s 664us/sample - loss: 0.4649 - accuracy: 0.8533 - val_loss: 0.4157 - val_accuracy: 0.8689\n",
      "Epoch 12/100\n",
      "559/559 [==============================] - 0s 664us/sample - loss: 0.4356 - accuracy: 0.8587 - val_loss: 0.3480 - val_accuracy: 0.8852\n",
      "Epoch 13/100\n",
      "559/559 [==============================] - 0s 665us/sample - loss: 0.3959 - accuracy: 0.8623 - val_loss: 0.3498 - val_accuracy: 0.8852\n",
      "Epoch 14/100\n",
      "559/559 [==============================] - 0s 666us/sample - loss: 0.3990 - accuracy: 0.8569 - val_loss: 0.3418 - val_accuracy: 0.8852\n",
      "Epoch 15/100\n",
      "559/559 [==============================] - 0s 667us/sample - loss: 0.3858 - accuracy: 0.8712 - val_loss: 0.3288 - val_accuracy: 0.9016\n",
      "Epoch 16/100\n",
      "559/559 [==============================] - 0s 666us/sample - loss: 0.4055 - accuracy: 0.8605 - val_loss: 0.3281 - val_accuracy: 0.9016\n",
      "Epoch 17/100\n",
      "559/559 [==============================] - 0s 716us/sample - loss: 0.3639 - accuracy: 0.8640 - val_loss: 0.3333 - val_accuracy: 0.8852\n",
      "Epoch 18/100\n",
      "559/559 [==============================] - 0s 718us/sample - loss: 0.3599 - accuracy: 0.8658 - val_loss: 0.3444 - val_accuracy: 0.9016\n",
      "Epoch 19/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.3381 - accuracy: 0.8748 - val_loss: 0.3520 - val_accuracy: 0.8852\n",
      "Epoch 20/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.3255 - accuracy: 0.8819 - val_loss: 0.3477 - val_accuracy: 0.8852\n",
      "Epoch 21/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.3107 - accuracy: 0.8837 - val_loss: 0.3231 - val_accuracy: 0.8852\n",
      "Epoch 22/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.3183 - accuracy: 0.8819 - val_loss: 0.3375 - val_accuracy: 0.8852\n",
      "Epoch 23/100\n",
      "559/559 [==============================] - 0s 663us/sample - loss: 0.3028 - accuracy: 0.8909 - val_loss: 0.3794 - val_accuracy: 0.8852\n",
      "Epoch 24/100\n",
      "559/559 [==============================] - 0s 664us/sample - loss: 0.3073 - accuracy: 0.8837 - val_loss: 0.3939 - val_accuracy: 0.8689\n",
      "Epoch 25/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.2845 - accuracy: 0.8891 - val_loss: 0.3413 - val_accuracy: 0.9016\n",
      "Epoch 26/100\n",
      "559/559 [==============================] - 0s 656us/sample - loss: 0.2777 - accuracy: 0.8819 - val_loss: 0.3472 - val_accuracy: 0.8852\n",
      "Epoch 27/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.2907 - accuracy: 0.8819 - val_loss: 0.4811 - val_accuracy: 0.8525\n",
      "Epoch 28/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.2489 - accuracy: 0.8998 - val_loss: 0.3343 - val_accuracy: 0.9016\n",
      "Epoch 29/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.2690 - accuracy: 0.8891 - val_loss: 0.4318 - val_accuracy: 0.8689\n",
      "Epoch 30/100\n",
      "559/559 [==============================] - 0s 662us/sample - loss: 0.2380 - accuracy: 0.9070 - val_loss: 0.3478 - val_accuracy: 0.8852\n",
      "Epoch 31/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.2566 - accuracy: 0.8980 - val_loss: 0.4536 - val_accuracy: 0.8197\n",
      "Epoch 32/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.2511 - accuracy: 0.8980 - val_loss: 0.3317 - val_accuracy: 0.9016\n",
      "Epoch 33/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.2276 - accuracy: 0.9070 - val_loss: 0.3224 - val_accuracy: 0.9016\n",
      "Epoch 34/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.2091 - accuracy: 0.9195 - val_loss: 0.3686 - val_accuracy: 0.8689\n",
      "Epoch 35/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.1907 - accuracy: 0.9320 - val_loss: 0.3595 - val_accuracy: 0.8689\n",
      "Epoch 36/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.2056 - accuracy: 0.9177 - val_loss: 0.3615 - val_accuracy: 0.8852\n",
      "Epoch 37/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.1773 - accuracy: 0.9392 - val_loss: 0.3898 - val_accuracy: 0.8689\n",
      "Epoch 38/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.1460 - accuracy: 0.9624 - val_loss: 0.3539 - val_accuracy: 0.9016\n",
      "Epoch 39/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.1695 - accuracy: 0.9374 - val_loss: 0.3603 - val_accuracy: 0.9016\n",
      "Epoch 40/100\n",
      "559/559 [==============================] - 0s 692us/sample - loss: 0.1501 - accuracy: 0.9481 - val_loss: 0.3844 - val_accuracy: 0.8852\n",
      "Epoch 41/100\n",
      "559/559 [==============================] - 0s 665us/sample - loss: 0.1817 - accuracy: 0.9356 - val_loss: 0.3939 - val_accuracy: 0.8361\n",
      "Epoch 42/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.1360 - accuracy: 0.9445 - val_loss: 0.4038 - val_accuracy: 0.8361\n",
      "Epoch 43/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.1388 - accuracy: 0.9392 - val_loss: 0.4227 - val_accuracy: 0.8197\n",
      "Epoch 44/100\n",
      "559/559 [==============================] - 0s 662us/sample - loss: 0.1102 - accuracy: 0.9624 - val_loss: 0.4414 - val_accuracy: 0.9016\n",
      "Epoch 45/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.1241 - accuracy: 0.9571 - val_loss: 0.3865 - val_accuracy: 0.8689\n",
      "Epoch 46/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.1093 - accuracy: 0.9589 - val_loss: 0.4068 - val_accuracy: 0.8525\n",
      "Epoch 47/100\n",
      "559/559 [==============================] - 0s 657us/sample - loss: 0.0987 - accuracy: 0.9642 - val_loss: 0.5094 - val_accuracy: 0.8852\n",
      "Epoch 48/100\n",
      "559/559 [==============================] - 0s 654us/sample - loss: 0.1133 - accuracy: 0.9571 - val_loss: 0.4654 - val_accuracy: 0.9016\n",
      "Epoch 49/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0892 - accuracy: 0.9678 - val_loss: 0.5703 - val_accuracy: 0.8852\n",
      "Epoch 50/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0940 - accuracy: 0.9660 - val_loss: 0.4239 - val_accuracy: 0.8689\n",
      "Epoch 51/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0844 - accuracy: 0.9750 - val_loss: 0.5620 - val_accuracy: 0.8033\n",
      "Epoch 52/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0810 - accuracy: 0.9696 - val_loss: 0.4456 - val_accuracy: 0.9016\n",
      "Epoch 53/100\n",
      "559/559 [==============================] - 0s 654us/sample - loss: 0.0989 - accuracy: 0.9624 - val_loss: 0.6790 - val_accuracy: 0.8852\n",
      "Epoch 54/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0927 - accuracy: 0.9714 - val_loss: 0.6897 - val_accuracy: 0.8852\n",
      "Epoch 55/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.1028 - accuracy: 0.9624 - val_loss: 0.6023 - val_accuracy: 0.7541\n",
      "Epoch 56/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0894 - accuracy: 0.9660 - val_loss: 0.4806 - val_accuracy: 0.8852\n",
      "Epoch 57/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0643 - accuracy: 0.9821 - val_loss: 0.4439 - val_accuracy: 0.8361\n",
      "Epoch 58/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0397 - accuracy: 0.9911 - val_loss: 0.5604 - val_accuracy: 0.9016\n",
      "Epoch 59/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0489 - accuracy: 0.9821 - val_loss: 0.5509 - val_accuracy: 0.8852\n",
      "Epoch 60/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0430 - accuracy: 0.9821 - val_loss: 0.5249 - val_accuracy: 0.8197\n",
      "Epoch 61/100\n",
      "559/559 [==============================] - 0s 657us/sample - loss: 0.0564 - accuracy: 0.9821 - val_loss: 0.7800 - val_accuracy: 0.7049\n",
      "Epoch 62/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0522 - accuracy: 0.9839 - val_loss: 0.6225 - val_accuracy: 0.8852\n",
      "Epoch 63/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0439 - accuracy: 0.9839 - val_loss: 0.6712 - val_accuracy: 0.7869\n",
      "Epoch 64/100\n",
      "559/559 [==============================] - 0s 657us/sample - loss: 0.0496 - accuracy: 0.9857 - val_loss: 0.6331 - val_accuracy: 0.8361\n",
      "Epoch 65/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0447 - accuracy: 0.9857 - val_loss: 0.5251 - val_accuracy: 0.8525\n",
      "Epoch 66/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0334 - accuracy: 0.9857 - val_loss: 0.5901 - val_accuracy: 0.8361\n",
      "Epoch 67/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0357 - accuracy: 0.9875 - val_loss: 0.6525 - val_accuracy: 0.8689\n",
      "Epoch 68/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0508 - accuracy: 0.9785 - val_loss: 0.6506 - val_accuracy: 0.8852\n",
      "Epoch 69/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0891 - accuracy: 0.9714 - val_loss: 0.7140 - val_accuracy: 0.7869\n",
      "Epoch 70/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0211 - accuracy: 0.9982 - val_loss: 0.5537 - val_accuracy: 0.9016\n",
      "Epoch 71/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0283 - accuracy: 0.9911 - val_loss: 0.5501 - val_accuracy: 0.8197\n",
      "Epoch 72/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0288 - accuracy: 0.9911 - val_loss: 0.7745 - val_accuracy: 0.8689\n",
      "Epoch 73/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0485 - accuracy: 0.9875 - val_loss: 0.7554 - val_accuracy: 0.8361\n",
      "Epoch 74/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0461 - accuracy: 0.9857 - val_loss: 0.8546 - val_accuracy: 0.8852\n",
      "Epoch 75/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0269 - accuracy: 0.9928 - val_loss: 0.8954 - val_accuracy: 0.8689\n",
      "Epoch 76/100\n",
      "559/559 [==============================] - 0s 654us/sample - loss: 0.0364 - accuracy: 0.9857 - val_loss: 0.6661 - val_accuracy: 0.8361\n",
      "Epoch 77/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0252 - accuracy: 0.9857 - val_loss: 0.6240 - val_accuracy: 0.8361\n",
      "Epoch 78/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0174 - accuracy: 0.9964 - val_loss: 0.5943 - val_accuracy: 0.8197\n",
      "Epoch 79/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0203 - accuracy: 0.9964 - val_loss: 0.6192 - val_accuracy: 0.8361\n",
      "Epoch 80/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0273 - accuracy: 0.9893 - val_loss: 0.6709 - val_accuracy: 0.8361\n",
      "Epoch 81/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0236 - accuracy: 0.9911 - val_loss: 0.6685 - val_accuracy: 0.8033\n",
      "Epoch 82/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0224 - accuracy: 0.9946 - val_loss: 0.6609 - val_accuracy: 0.8361\n",
      "Epoch 83/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0159 - accuracy: 0.9946 - val_loss: 0.6707 - val_accuracy: 0.8689\n",
      "Epoch 84/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0189 - accuracy: 0.9928 - val_loss: 0.7300 - val_accuracy: 0.9016\n",
      "Epoch 85/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0100 - accuracy: 1.0000 - val_loss: 0.8000 - val_accuracy: 0.9016\n",
      "Epoch 86/100\n",
      "559/559 [==============================] - 0s 657us/sample - loss: 0.0165 - accuracy: 0.9928 - val_loss: 0.6861 - val_accuracy: 0.8361\n",
      "Epoch 87/100\n",
      "559/559 [==============================] - 0s 664us/sample - loss: 0.0141 - accuracy: 0.9964 - val_loss: 0.6695 - val_accuracy: 0.8689\n",
      "Epoch 88/100\n",
      "559/559 [==============================] - 0s 660us/sample - loss: 0.0206 - accuracy: 0.9946 - val_loss: 0.6822 - val_accuracy: 0.9016\n",
      "Epoch 89/100\n",
      "559/559 [==============================] - 0s 666us/sample - loss: 0.0201 - accuracy: 0.9946 - val_loss: 0.8156 - val_accuracy: 0.8197\n",
      "Epoch 90/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0168 - accuracy: 0.9946 - val_loss: 0.6901 - val_accuracy: 0.8361\n",
      "Epoch 91/100\n",
      "559/559 [==============================] - 0s 654us/sample - loss: 0.0127 - accuracy: 0.9982 - val_loss: 0.6944 - val_accuracy: 0.8361\n",
      "Epoch 92/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0147 - accuracy: 0.9946 - val_loss: 0.7857 - val_accuracy: 0.8689\n",
      "Epoch 93/100\n",
      "559/559 [==============================] - 0s 658us/sample - loss: 0.0189 - accuracy: 0.9928 - val_loss: 0.7549 - val_accuracy: 0.8852\n",
      "Epoch 94/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0111 - accuracy: 0.9964 - val_loss: 0.8583 - val_accuracy: 0.7869\n",
      "Epoch 95/100\n",
      "559/559 [==============================] - 0s 662us/sample - loss: 0.0281 - accuracy: 0.9911 - val_loss: 0.6569 - val_accuracy: 0.8197\n",
      "Epoch 96/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0143 - accuracy: 0.9946 - val_loss: 0.9484 - val_accuracy: 0.7869\n",
      "Epoch 97/100\n",
      "559/559 [==============================] - 0s 655us/sample - loss: 0.0230 - accuracy: 0.9911 - val_loss: 1.1946 - val_accuracy: 0.8852\n",
      "Epoch 98/100\n",
      "559/559 [==============================] - 0s 659us/sample - loss: 0.0230 - accuracy: 0.9928 - val_loss: 0.7136 - val_accuracy: 0.8361\n",
      "Epoch 99/100\n",
      "559/559 [==============================] - 0s 661us/sample - loss: 0.0152 - accuracy: 0.9946 - val_loss: 0.7863 - val_accuracy: 0.8197\n",
      "Epoch 100/100\n",
      "559/559 [==============================] - 0s 648us/sample - loss: 0.0091 - accuracy: 0.9982 - val_loss: 0.8773 - val_accuracy: 0.8361\n",
      "accuracy for model 7 is 83.60655903816223\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_42 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_35 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_21 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_36 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_37 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_45 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_38 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_46 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_39 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_47 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 560 samples, validate on 60 samples\n",
      "Epoch 1/100\n",
      "560/560 [==============================] - 1s 1ms/sample - loss: 0.9130 - accuracy: 0.6911 - val_loss: 7.6328 - val_accuracy: 0.6833\n",
      "Epoch 2/100\n",
      "560/560 [==============================] - 0s 640us/sample - loss: 0.6487 - accuracy: 0.8089 - val_loss: 4.3246 - val_accuracy: 0.7000\n",
      "Epoch 3/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.5663 - accuracy: 0.8286 - val_loss: 1.2227 - val_accuracy: 0.7833\n",
      "Epoch 4/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.4993 - accuracy: 0.8536 - val_loss: 1.0511 - val_accuracy: 0.7333\n",
      "Epoch 5/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.4456 - accuracy: 0.8607 - val_loss: 0.5180 - val_accuracy: 0.8500\n",
      "Epoch 6/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.4419 - accuracy: 0.8500 - val_loss: 0.5215 - val_accuracy: 0.8167\n",
      "Epoch 7/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.3948 - accuracy: 0.8643 - val_loss: 0.6665 - val_accuracy: 0.8500\n",
      "Epoch 8/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.3550 - accuracy: 0.8786 - val_loss: 0.5692 - val_accuracy: 0.8333\n",
      "Epoch 9/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.3522 - accuracy: 0.8804 - val_loss: 0.5478 - val_accuracy: 0.8833\n",
      "Epoch 10/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.3470 - accuracy: 0.8714 - val_loss: 0.6050 - val_accuracy: 0.8500\n",
      "Epoch 11/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3374 - accuracy: 0.8696 - val_loss: 0.6276 - val_accuracy: 0.8167\n",
      "Epoch 12/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.2881 - accuracy: 0.8893 - val_loss: 0.7411 - val_accuracy: 0.7833\n",
      "Epoch 13/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.2990 - accuracy: 0.8804 - val_loss: 0.6853 - val_accuracy: 0.8500\n",
      "Epoch 14/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.3003 - accuracy: 0.8714 - val_loss: 0.5579 - val_accuracy: 0.8500\n",
      "Epoch 15/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2885 - accuracy: 0.8982 - val_loss: 0.5860 - val_accuracy: 0.8333\n",
      "Epoch 16/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.2634 - accuracy: 0.8982 - val_loss: 0.5722 - val_accuracy: 0.8333\n",
      "Epoch 17/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2375 - accuracy: 0.8946 - val_loss: 0.6773 - val_accuracy: 0.8167\n",
      "Epoch 18/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2239 - accuracy: 0.9054 - val_loss: 0.6733 - val_accuracy: 0.8167\n",
      "Epoch 19/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.2265 - accuracy: 0.9232 - val_loss: 0.6124 - val_accuracy: 0.8000\n",
      "Epoch 20/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2003 - accuracy: 0.9196 - val_loss: 0.6331 - val_accuracy: 0.7667\n",
      "Epoch 21/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2108 - accuracy: 0.9143 - val_loss: 0.6114 - val_accuracy: 0.8000\n",
      "Epoch 22/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1728 - accuracy: 0.9393 - val_loss: 0.6604 - val_accuracy: 0.8000\n",
      "Epoch 23/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.1868 - accuracy: 0.9304 - val_loss: 0.6578 - val_accuracy: 0.8000\n",
      "Epoch 24/100\n",
      "560/560 [==============================] - 0s 640us/sample - loss: 0.1748 - accuracy: 0.9375 - val_loss: 0.6583 - val_accuracy: 0.7667\n",
      "Epoch 25/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1368 - accuracy: 0.9518 - val_loss: 0.6512 - val_accuracy: 0.7333\n",
      "Epoch 26/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1432 - accuracy: 0.9518 - val_loss: 0.6759 - val_accuracy: 0.7500\n",
      "Epoch 27/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.1429 - accuracy: 0.9446 - val_loss: 0.7333 - val_accuracy: 0.7167\n",
      "Epoch 28/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.1659 - accuracy: 0.9500 - val_loss: 0.6634 - val_accuracy: 0.8000\n",
      "Epoch 29/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1113 - accuracy: 0.9679 - val_loss: 0.7040 - val_accuracy: 0.7333\n",
      "Epoch 30/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1205 - accuracy: 0.9589 - val_loss: 0.8228 - val_accuracy: 0.7833\n",
      "Epoch 31/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0975 - accuracy: 0.9696 - val_loss: 0.7795 - val_accuracy: 0.7333\n",
      "Epoch 32/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0994 - accuracy: 0.9714 - val_loss: 0.8242 - val_accuracy: 0.7667\n",
      "Epoch 33/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0908 - accuracy: 0.9661 - val_loss: 0.8405 - val_accuracy: 0.7833\n",
      "Epoch 34/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0856 - accuracy: 0.9696 - val_loss: 0.9936 - val_accuracy: 0.7667\n",
      "Epoch 35/100\n",
      "560/560 [==============================] - 0s 640us/sample - loss: 0.0588 - accuracy: 0.9875 - val_loss: 0.8435 - val_accuracy: 0.7500\n",
      "Epoch 36/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0903 - accuracy: 0.9625 - val_loss: 0.9464 - val_accuracy: 0.8000\n",
      "Epoch 37/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0707 - accuracy: 0.9804 - val_loss: 0.9977 - val_accuracy: 0.7833\n",
      "Epoch 38/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0588 - accuracy: 0.9804 - val_loss: 0.9247 - val_accuracy: 0.7333\n",
      "Epoch 39/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0561 - accuracy: 0.9839 - val_loss: 0.9949 - val_accuracy: 0.7833\n",
      "Epoch 40/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0462 - accuracy: 0.9875 - val_loss: 0.9712 - val_accuracy: 0.7667\n",
      "Epoch 41/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0545 - accuracy: 0.9786 - val_loss: 0.8767 - val_accuracy: 0.7667\n",
      "Epoch 42/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0446 - accuracy: 0.9893 - val_loss: 1.1012 - val_accuracy: 0.6833\n",
      "Epoch 43/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0494 - accuracy: 0.9839 - val_loss: 0.9668 - val_accuracy: 0.8167\n",
      "Epoch 44/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0362 - accuracy: 0.9911 - val_loss: 1.2315 - val_accuracy: 0.6833\n",
      "Epoch 45/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0413 - accuracy: 0.9839 - val_loss: 1.1081 - val_accuracy: 0.7500\n",
      "Epoch 46/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0424 - accuracy: 0.9839 - val_loss: 1.0782 - val_accuracy: 0.7500\n",
      "Epoch 47/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0441 - accuracy: 0.9839 - val_loss: 1.2356 - val_accuracy: 0.7667\n",
      "Epoch 48/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0252 - accuracy: 0.9929 - val_loss: 1.1563 - val_accuracy: 0.7167\n",
      "Epoch 49/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0244 - accuracy: 0.9946 - val_loss: 1.1360 - val_accuracy: 0.7833\n",
      "Epoch 50/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0319 - accuracy: 0.9929 - val_loss: 1.2951 - val_accuracy: 0.7833\n",
      "Epoch 51/100\n",
      "560/560 [==============================] - 0s 738us/sample - loss: 0.0245 - accuracy: 0.9911 - val_loss: 1.0509 - val_accuracy: 0.7667\n",
      "Epoch 52/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0185 - accuracy: 0.9964 - val_loss: 1.1773 - val_accuracy: 0.7500\n",
      "Epoch 53/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0269 - accuracy: 0.9929 - val_loss: 1.0475 - val_accuracy: 0.7833\n",
      "Epoch 54/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0229 - accuracy: 0.9929 - val_loss: 1.1468 - val_accuracy: 0.7500\n",
      "Epoch 55/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0342 - accuracy: 0.9911 - val_loss: 1.1755 - val_accuracy: 0.8000\n",
      "Epoch 56/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0290 - accuracy: 0.9929 - val_loss: 1.0678 - val_accuracy: 0.7500\n",
      "Epoch 57/100\n",
      "560/560 [==============================] - 0s 638us/sample - loss: 0.0159 - accuracy: 0.9964 - val_loss: 1.2158 - val_accuracy: 0.7500\n",
      "Epoch 58/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0244 - accuracy: 0.9893 - val_loss: 1.2466 - val_accuracy: 0.7833\n",
      "Epoch 59/100\n",
      "560/560 [==============================] - 0s 640us/sample - loss: 0.0284 - accuracy: 0.9911 - val_loss: 1.4963 - val_accuracy: 0.7000\n",
      "Epoch 60/100\n",
      "560/560 [==============================] - 0s 644us/sample - loss: 0.0154 - accuracy: 0.9946 - val_loss: 1.5005 - val_accuracy: 0.8000\n",
      "Epoch 61/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0312 - accuracy: 0.9911 - val_loss: 1.2827 - val_accuracy: 0.7833\n",
      "Epoch 62/100\n",
      "560/560 [==============================] - 0s 641us/sample - loss: 0.0319 - accuracy: 0.9875 - val_loss: 1.2679 - val_accuracy: 0.7667\n",
      "Epoch 63/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0207 - accuracy: 0.9946 - val_loss: 1.0404 - val_accuracy: 0.8000\n",
      "Epoch 64/100\n",
      "560/560 [==============================] - 0s 643us/sample - loss: 0.0226 - accuracy: 0.9911 - val_loss: 1.1911 - val_accuracy: 0.7500\n",
      "Epoch 65/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0150 - accuracy: 0.9946 - val_loss: 1.2683 - val_accuracy: 0.7833\n",
      "Epoch 66/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0162 - accuracy: 0.9964 - val_loss: 1.1629 - val_accuracy: 0.7667\n",
      "Epoch 67/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0146 - accuracy: 0.9946 - val_loss: 1.2575 - val_accuracy: 0.7833\n",
      "Epoch 68/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0096 - accuracy: 0.9964 - val_loss: 1.2951 - val_accuracy: 0.7833\n",
      "Epoch 69/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0121 - accuracy: 0.9982 - val_loss: 1.2120 - val_accuracy: 0.8000\n",
      "Epoch 70/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0062 - accuracy: 1.0000 - val_loss: 1.4801 - val_accuracy: 0.7667\n",
      "Epoch 71/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0136 - accuracy: 0.9964 - val_loss: 1.0635 - val_accuracy: 0.7667\n",
      "Epoch 72/100\n",
      "560/560 [==============================] - 0s 641us/sample - loss: 0.0190 - accuracy: 0.9929 - val_loss: 1.1749 - val_accuracy: 0.7167\n",
      "Epoch 73/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0264 - accuracy: 0.9911 - val_loss: 1.1282 - val_accuracy: 0.7667\n",
      "Epoch 74/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0222 - accuracy: 0.9911 - val_loss: 1.6583 - val_accuracy: 0.7833\n",
      "Epoch 75/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0468 - accuracy: 0.9875 - val_loss: 1.5213 - val_accuracy: 0.7833\n",
      "Epoch 76/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0111 - accuracy: 0.9964 - val_loss: 1.1220 - val_accuracy: 0.8333\n",
      "Epoch 77/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0229 - accuracy: 0.9893 - val_loss: 1.0490 - val_accuracy: 0.7500\n",
      "Epoch 78/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0132 - accuracy: 0.9982 - val_loss: 0.9604 - val_accuracy: 0.8000\n",
      "Epoch 79/100\n",
      "560/560 [==============================] - 0s 660us/sample - loss: 0.0152 - accuracy: 0.9946 - val_loss: 1.1515 - val_accuracy: 0.7833\n",
      "Epoch 80/100\n",
      "560/560 [==============================] - 0s 649us/sample - loss: 0.0171 - accuracy: 0.9964 - val_loss: 1.2281 - val_accuracy: 0.7833\n",
      "Epoch 81/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0062 - accuracy: 0.9982 - val_loss: 1.3149 - val_accuracy: 0.7667\n",
      "Epoch 82/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0056 - accuracy: 0.9982 - val_loss: 1.2805 - val_accuracy: 0.7667\n",
      "Epoch 83/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0164 - accuracy: 0.9929 - val_loss: 1.0862 - val_accuracy: 0.8000\n",
      "Epoch 84/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0059 - accuracy: 1.0000 - val_loss: 1.3138 - val_accuracy: 0.7833\n",
      "Epoch 85/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0033 - accuracy: 1.0000 - val_loss: 1.2939 - val_accuracy: 0.8000\n",
      "Epoch 86/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0063 - accuracy: 1.0000 - val_loss: 1.2743 - val_accuracy: 0.8000\n",
      "Epoch 87/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0029 - accuracy: 1.0000 - val_loss: 1.1158 - val_accuracy: 0.8000\n",
      "Epoch 88/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0029 - accuracy: 1.0000 - val_loss: 1.3258 - val_accuracy: 0.7833\n",
      "Epoch 89/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0047 - accuracy: 1.0000 - val_loss: 1.3162 - val_accuracy: 0.7833\n",
      "Epoch 90/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0019 - accuracy: 1.0000 - val_loss: 1.2207 - val_accuracy: 0.8000\n",
      "Epoch 91/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0054 - accuracy: 0.9982 - val_loss: 1.3301 - val_accuracy: 0.7833\n",
      "Epoch 92/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0216 - accuracy: 0.9929 - val_loss: 1.3410 - val_accuracy: 0.7833\n",
      "Epoch 93/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0283 - accuracy: 0.9929 - val_loss: 1.6399 - val_accuracy: 0.6167\n",
      "Epoch 94/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0093 - accuracy: 0.9982 - val_loss: 1.3084 - val_accuracy: 0.7833\n",
      "Epoch 95/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0056 - accuracy: 0.9982 - val_loss: 1.1705 - val_accuracy: 0.7333\n",
      "Epoch 96/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0225 - accuracy: 0.9911 - val_loss: 1.7131 - val_accuracy: 0.6833\n",
      "Epoch 97/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0126 - accuracy: 0.9982 - val_loss: 1.4906 - val_accuracy: 0.7667\n",
      "Epoch 98/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0178 - accuracy: 0.9929 - val_loss: 1.2596 - val_accuracy: 0.7667\n",
      "Epoch 99/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0093 - accuracy: 0.9964 - val_loss: 1.4087 - val_accuracy: 0.8000\n",
      "Epoch 100/100\n",
      "560/560 [==============================] - 0s 624us/sample - loss: 0.0071 - accuracy: 0.9982 - val_loss: 1.4041 - val_accuracy: 0.7833\n",
      "accuracy for model 8 is 78.33333611488342\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [1 0 0 0]\n",
      " [0 0 1 0]\n",
      " [0 0 1 0]]\n",
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_48 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_40 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_24 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_49 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_41 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_25 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_50 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_42 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_26 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_51 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_43 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_52 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_44 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_53 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 560 samples, validate on 60 samples\n",
      "Epoch 1/100\n",
      "560/560 [==============================] - 1s 1ms/sample - loss: 0.8390 - accuracy: 0.7232 - val_loss: 3.0271 - val_accuracy: 0.7000\n",
      "Epoch 2/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.5978 - accuracy: 0.8214 - val_loss: 2.8419 - val_accuracy: 0.7000\n",
      "Epoch 3/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.5343 - accuracy: 0.8446 - val_loss: 1.3479 - val_accuracy: 0.7000\n",
      "Epoch 4/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.4849 - accuracy: 0.8589 - val_loss: 0.9569 - val_accuracy: 0.7167\n",
      "Epoch 5/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.4740 - accuracy: 0.8554 - val_loss: 0.9626 - val_accuracy: 0.7167\n",
      "Epoch 6/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.4341 - accuracy: 0.8679 - val_loss: 0.7232 - val_accuracy: 0.7833\n",
      "Epoch 7/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.4317 - accuracy: 0.8625 - val_loss: 0.6654 - val_accuracy: 0.7667\n",
      "Epoch 8/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.4014 - accuracy: 0.8732 - val_loss: 0.8844 - val_accuracy: 0.7833\n",
      "Epoch 9/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.3734 - accuracy: 0.8768 - val_loss: 0.6698 - val_accuracy: 0.8167\n",
      "Epoch 10/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.3392 - accuracy: 0.8929 - val_loss: 0.6460 - val_accuracy: 0.7833\n",
      "Epoch 11/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.3063 - accuracy: 0.9036 - val_loss: 0.6640 - val_accuracy: 0.7833\n",
      "Epoch 12/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.3189 - accuracy: 0.8929 - val_loss: 0.7621 - val_accuracy: 0.8333\n",
      "Epoch 13/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2821 - accuracy: 0.9143 - val_loss: 0.7270 - val_accuracy: 0.8000\n",
      "Epoch 14/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2691 - accuracy: 0.9161 - val_loss: 0.7203 - val_accuracy: 0.8167\n",
      "Epoch 15/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2458 - accuracy: 0.9321 - val_loss: 0.9273 - val_accuracy: 0.7667\n",
      "Epoch 16/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.2554 - accuracy: 0.9107 - val_loss: 0.9961 - val_accuracy: 0.7667\n",
      "Epoch 17/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2276 - accuracy: 0.9125 - val_loss: 0.7503 - val_accuracy: 0.7667\n",
      "Epoch 18/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1920 - accuracy: 0.9375 - val_loss: 0.7082 - val_accuracy: 0.7833\n",
      "Epoch 19/100\n",
      "560/560 [==============================] - 0s 637us/sample - loss: 0.1623 - accuracy: 0.9554 - val_loss: 0.7191 - val_accuracy: 0.8333\n",
      "Epoch 20/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1548 - accuracy: 0.9464 - val_loss: 0.8113 - val_accuracy: 0.7500\n",
      "Epoch 21/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1642 - accuracy: 0.9464 - val_loss: 0.8443 - val_accuracy: 0.7667\n",
      "Epoch 22/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1503 - accuracy: 0.9536 - val_loss: 0.8330 - val_accuracy: 0.7667\n",
      "Epoch 23/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1317 - accuracy: 0.9589 - val_loss: 0.8177 - val_accuracy: 0.7667\n",
      "Epoch 24/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1138 - accuracy: 0.9679 - val_loss: 0.7752 - val_accuracy: 0.8167\n",
      "Epoch 25/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1131 - accuracy: 0.9696 - val_loss: 0.9297 - val_accuracy: 0.7667\n",
      "Epoch 26/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.1280 - accuracy: 0.9625 - val_loss: 0.9971 - val_accuracy: 0.7333\n",
      "Epoch 27/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.1017 - accuracy: 0.9661 - val_loss: 1.1507 - val_accuracy: 0.7500\n",
      "Epoch 28/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0993 - accuracy: 0.9625 - val_loss: 1.0326 - val_accuracy: 0.7333\n",
      "Epoch 29/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0872 - accuracy: 0.9786 - val_loss: 0.9571 - val_accuracy: 0.7667\n",
      "Epoch 30/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0831 - accuracy: 0.9786 - val_loss: 0.9912 - val_accuracy: 0.7500\n",
      "Epoch 31/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0694 - accuracy: 0.9804 - val_loss: 1.0521 - val_accuracy: 0.7667\n",
      "Epoch 32/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0530 - accuracy: 0.9857 - val_loss: 1.1132 - val_accuracy: 0.7833\n",
      "Epoch 33/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0684 - accuracy: 0.9750 - val_loss: 0.9480 - val_accuracy: 0.7500\n",
      "Epoch 34/100\n",
      "560/560 [==============================] - 0s 755us/sample - loss: 0.0471 - accuracy: 0.9929 - val_loss: 1.3022 - val_accuracy: 0.7333\n",
      "Epoch 35/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0515 - accuracy: 0.9875 - val_loss: 1.0823 - val_accuracy: 0.7333\n",
      "Epoch 36/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0635 - accuracy: 0.9804 - val_loss: 1.0167 - val_accuracy: 0.8167\n",
      "Epoch 37/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0659 - accuracy: 0.9732 - val_loss: 1.1990 - val_accuracy: 0.7500\n",
      "Epoch 38/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0829 - accuracy: 0.9732 - val_loss: 1.7894 - val_accuracy: 0.7500\n",
      "Epoch 39/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0543 - accuracy: 0.9857 - val_loss: 1.2488 - val_accuracy: 0.7667\n",
      "Epoch 40/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0519 - accuracy: 0.9839 - val_loss: 1.4777 - val_accuracy: 0.7500\n",
      "Epoch 41/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0462 - accuracy: 0.9857 - val_loss: 1.3085 - val_accuracy: 0.7333\n",
      "Epoch 42/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0389 - accuracy: 0.9857 - val_loss: 1.1292 - val_accuracy: 0.7833\n",
      "Epoch 43/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0446 - accuracy: 0.9875 - val_loss: 1.2719 - val_accuracy: 0.7167\n",
      "Epoch 44/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0274 - accuracy: 0.9946 - val_loss: 1.5783 - val_accuracy: 0.8000\n",
      "Epoch 45/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0831 - accuracy: 0.9732 - val_loss: 1.4001 - val_accuracy: 0.7167\n",
      "Epoch 46/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0572 - accuracy: 0.9804 - val_loss: 1.4921 - val_accuracy: 0.7167\n",
      "Epoch 47/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0258 - accuracy: 0.9929 - val_loss: 1.6884 - val_accuracy: 0.7333\n",
      "Epoch 48/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0307 - accuracy: 0.9893 - val_loss: 1.5136 - val_accuracy: 0.7333\n",
      "Epoch 49/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0266 - accuracy: 0.9929 - val_loss: 1.3875 - val_accuracy: 0.7500\n",
      "Epoch 50/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0304 - accuracy: 0.9929 - val_loss: 1.5945 - val_accuracy: 0.7333\n",
      "Epoch 51/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0254 - accuracy: 0.9929 - val_loss: 1.3729 - val_accuracy: 0.7333\n",
      "Epoch 52/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0231 - accuracy: 0.9911 - val_loss: 1.3931 - val_accuracy: 0.7167\n",
      "Epoch 53/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0282 - accuracy: 0.9893 - val_loss: 1.4568 - val_accuracy: 0.7333\n",
      "Epoch 54/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0180 - accuracy: 0.9946 - val_loss: 1.3075 - val_accuracy: 0.7500\n",
      "Epoch 55/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0228 - accuracy: 0.9946 - val_loss: 1.4580 - val_accuracy: 0.7167\n",
      "Epoch 56/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0247 - accuracy: 0.9911 - val_loss: 1.4667 - val_accuracy: 0.7333\n",
      "Epoch 57/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0270 - accuracy: 0.9929 - val_loss: 1.6368 - val_accuracy: 0.7000\n",
      "Epoch 58/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0174 - accuracy: 0.9964 - val_loss: 1.4062 - val_accuracy: 0.7667\n",
      "Epoch 59/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0210 - accuracy: 0.9946 - val_loss: 1.4429 - val_accuracy: 0.7333\n",
      "Epoch 60/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0200 - accuracy: 0.9911 - val_loss: 1.4077 - val_accuracy: 0.7500\n",
      "Epoch 61/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0112 - accuracy: 0.9964 - val_loss: 1.3462 - val_accuracy: 0.7667\n",
      "Epoch 62/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0191 - accuracy: 0.9964 - val_loss: 1.5998 - val_accuracy: 0.7333\n",
      "Epoch 63/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0184 - accuracy: 0.9964 - val_loss: 1.4695 - val_accuracy: 0.7500\n",
      "Epoch 64/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0145 - accuracy: 0.9964 - val_loss: 1.7958 - val_accuracy: 0.7167\n",
      "Epoch 65/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0093 - accuracy: 1.0000 - val_loss: 1.6225 - val_accuracy: 0.7500\n",
      "Epoch 66/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0146 - accuracy: 0.9964 - val_loss: 1.4505 - val_accuracy: 0.7167\n",
      "Epoch 67/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0144 - accuracy: 0.9929 - val_loss: 1.6034 - val_accuracy: 0.7167\n",
      "Epoch 68/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0245 - accuracy: 0.9929 - val_loss: 1.6666 - val_accuracy: 0.6833\n",
      "Epoch 69/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0429 - accuracy: 0.9804 - val_loss: 1.6194 - val_accuracy: 0.7167\n",
      "Epoch 70/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0286 - accuracy: 0.9875 - val_loss: 1.8184 - val_accuracy: 0.7167\n",
      "Epoch 71/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0125 - accuracy: 0.9982 - val_loss: 1.8603 - val_accuracy: 0.7333\n",
      "Epoch 72/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0076 - accuracy: 1.0000 - val_loss: 1.6692 - val_accuracy: 0.7167\n",
      "Epoch 73/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0165 - accuracy: 0.9946 - val_loss: 1.6660 - val_accuracy: 0.7000\n",
      "Epoch 74/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0144 - accuracy: 0.9964 - val_loss: 1.6331 - val_accuracy: 0.7500\n",
      "Epoch 75/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0155 - accuracy: 0.9946 - val_loss: 1.6811 - val_accuracy: 0.7333\n",
      "Epoch 76/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0185 - accuracy: 0.9946 - val_loss: 1.6446 - val_accuracy: 0.7000\n",
      "Epoch 77/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0075 - accuracy: 0.9982 - val_loss: 1.4667 - val_accuracy: 0.7833\n",
      "Epoch 78/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0496 - accuracy: 0.9821 - val_loss: 1.5474 - val_accuracy: 0.7500\n",
      "Epoch 79/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0488 - accuracy: 0.9857 - val_loss: 1.2663 - val_accuracy: 0.7667\n",
      "Epoch 80/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0238 - accuracy: 0.9911 - val_loss: 1.2131 - val_accuracy: 0.7500\n",
      "Epoch 81/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0240 - accuracy: 0.9911 - val_loss: 1.4672 - val_accuracy: 0.7500\n",
      "Epoch 82/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0310 - accuracy: 0.9929 - val_loss: 1.7485 - val_accuracy: 0.6667\n",
      "Epoch 83/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0281 - accuracy: 0.9929 - val_loss: 1.8331 - val_accuracy: 0.7667\n",
      "Epoch 84/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0208 - accuracy: 0.9929 - val_loss: 1.6044 - val_accuracy: 0.7667\n",
      "Epoch 85/100\n",
      "560/560 [==============================] - 7s 12ms/sample - loss: 0.0200 - accuracy: 0.9929 - val_loss: 1.6786 - val_accuracy: 0.7500\n",
      "Epoch 86/100\n",
      "560/560 [==============================] - 0s 822us/sample - loss: 0.0126 - accuracy: 0.9964 - val_loss: 1.8868 - val_accuracy: 0.7167\n",
      "Epoch 87/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0100 - accuracy: 0.9964 - val_loss: 1.6641 - val_accuracy: 0.7333\n",
      "Epoch 88/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0264 - accuracy: 0.9946 - val_loss: 1.9955 - val_accuracy: 0.7333\n",
      "Epoch 89/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0067 - accuracy: 1.0000 - val_loss: 1.8950 - val_accuracy: 0.7167\n",
      "Epoch 90/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0047 - accuracy: 1.0000 - val_loss: 1.7151 - val_accuracy: 0.7167\n",
      "Epoch 91/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0201 - accuracy: 0.9929 - val_loss: 1.6974 - val_accuracy: 0.7167\n",
      "Epoch 92/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0108 - accuracy: 0.9964 - val_loss: 2.0348 - val_accuracy: 0.7167\n",
      "Epoch 93/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0075 - accuracy: 0.9982 - val_loss: 1.6851 - val_accuracy: 0.7167\n",
      "Epoch 94/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0399 - accuracy: 0.9893 - val_loss: 2.0875 - val_accuracy: 0.7333\n",
      "Epoch 95/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0185 - accuracy: 0.9946 - val_loss: 1.8241 - val_accuracy: 0.6833\n",
      "Epoch 96/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0163 - accuracy: 0.9946 - val_loss: 1.8802 - val_accuracy: 0.7667\n",
      "Epoch 97/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0166 - accuracy: 0.9964 - val_loss: 1.6068 - val_accuracy: 0.7333\n",
      "Epoch 98/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0082 - accuracy: 0.9982 - val_loss: 1.5951 - val_accuracy: 0.7167\n",
      "Epoch 99/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0040 - accuracy: 1.0000 - val_loss: 1.7484 - val_accuracy: 0.7500\n",
      "Epoch 100/100\n",
      "560/560 [==============================] - 0s 623us/sample - loss: 0.0067 - accuracy: 1.0000 - val_loss: 1.7334 - val_accuracy: 0.7167\n",
      "accuracy for model 9 is 71.66666388511658\n",
      "(620, 4)\n",
      "[[0 0 1 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 0]\n",
      " ...\n",
      " [0 0 0 1]\n",
      " [0 1 0 0]\n",
      " [0 1 0 0]]\n",
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_54 (Dense)             (None, 192)               24147264  \n",
      "_________________________________________________________________\n",
      "activation_45 (Activation)   (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dropout_27 (Dropout)         (None, 192)               0         \n",
      "_________________________________________________________________\n",
      "dense_55 (Dense)             (None, 128)               24704     \n",
      "_________________________________________________________________\n",
      "activation_46 (Activation)   (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_28 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_56 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "activation_47 (Activation)   (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_29 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_57 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "activation_48 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_58 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_49 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_59 (Dense)             (None, 4)                 68        \n",
      "=================================================================\n",
      "Total params: 24,183,028\n",
      "Trainable params: 24,182,964\n",
      "Non-trainable params: 64\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 560 samples, validate on 60 samples\n",
      "Epoch 1/100\n",
      "560/560 [==============================] - 18s 33ms/sample - loss: 1.0092 - accuracy: 0.6446 - val_loss: 1.8235 - val_accuracy: 0.7667\n",
      "Epoch 2/100\n",
      "560/560 [==============================] - 0s 647us/sample - loss: 0.6361 - accuracy: 0.8339 - val_loss: 1.7900 - val_accuracy: 0.7667\n",
      "Epoch 3/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.5247 - accuracy: 0.8518 - val_loss: 0.8742 - val_accuracy: 0.8333\n",
      "Epoch 4/100\n",
      "560/560 [==============================] - 0s 642us/sample - loss: 0.4675 - accuracy: 0.8571 - val_loss: 0.5869 - val_accuracy: 0.8833\n",
      "Epoch 5/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.4413 - accuracy: 0.8518 - val_loss: 0.9839 - val_accuracy: 0.7833\n",
      "Epoch 6/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.4150 - accuracy: 0.8661 - val_loss: 0.9439 - val_accuracy: 0.7167\n",
      "Epoch 7/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3902 - accuracy: 0.8679 - val_loss: 1.2226 - val_accuracy: 0.6000\n",
      "Epoch 8/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3421 - accuracy: 0.8786 - val_loss: 0.6099 - val_accuracy: 0.8500\n",
      "Epoch 9/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3352 - accuracy: 0.8714 - val_loss: 0.6071 - val_accuracy: 0.8500\n",
      "Epoch 10/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.3190 - accuracy: 0.8750 - val_loss: 0.6934 - val_accuracy: 0.8667\n",
      "Epoch 11/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3424 - accuracy: 0.8804 - val_loss: 0.7159 - val_accuracy: 0.8333\n",
      "Epoch 12/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.3003 - accuracy: 0.8839 - val_loss: 0.7808 - val_accuracy: 0.8000\n",
      "Epoch 13/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.2843 - accuracy: 0.8875 - val_loss: 0.6752 - val_accuracy: 0.8167\n",
      "Epoch 14/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.2808 - accuracy: 0.8911 - val_loss: 0.6707 - val_accuracy: 0.8000\n",
      "Epoch 15/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.2943 - accuracy: 0.8804 - val_loss: 0.5770 - val_accuracy: 0.8500\n",
      "Epoch 16/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.2794 - accuracy: 0.9054 - val_loss: 0.4801 - val_accuracy: 0.8667\n",
      "Epoch 17/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.2534 - accuracy: 0.9036 - val_loss: 0.6123 - val_accuracy: 0.8333\n",
      "Epoch 18/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.2123 - accuracy: 0.9196 - val_loss: 0.4807 - val_accuracy: 0.8500\n",
      "Epoch 19/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.2331 - accuracy: 0.9107 - val_loss: 0.4916 - val_accuracy: 0.8333\n",
      "Epoch 20/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.2214 - accuracy: 0.9179 - val_loss: 0.5212 - val_accuracy: 0.8500\n",
      "Epoch 21/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.2068 - accuracy: 0.9232 - val_loss: 0.5035 - val_accuracy: 0.8333\n",
      "Epoch 22/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1823 - accuracy: 0.9321 - val_loss: 0.5081 - val_accuracy: 0.8667\n",
      "Epoch 23/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.1709 - accuracy: 0.9375 - val_loss: 0.4813 - val_accuracy: 0.8333\n",
      "Epoch 24/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1640 - accuracy: 0.9464 - val_loss: 0.5354 - val_accuracy: 0.8667\n",
      "Epoch 25/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1915 - accuracy: 0.9304 - val_loss: 0.5578 - val_accuracy: 0.8167\n",
      "Epoch 26/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.1487 - accuracy: 0.9500 - val_loss: 0.5783 - val_accuracy: 0.8500\n",
      "Epoch 27/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1360 - accuracy: 0.9536 - val_loss: 0.5774 - val_accuracy: 0.8500\n",
      "Epoch 28/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.1057 - accuracy: 0.9696 - val_loss: 0.5756 - val_accuracy: 0.8333\n",
      "Epoch 29/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.1048 - accuracy: 0.9714 - val_loss: 0.6367 - val_accuracy: 0.8167\n",
      "Epoch 30/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0970 - accuracy: 0.9714 - val_loss: 0.6281 - val_accuracy: 0.7833\n",
      "Epoch 31/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.1219 - accuracy: 0.9625 - val_loss: 0.7350 - val_accuracy: 0.8167\n",
      "Epoch 32/100\n",
      "560/560 [==============================] - 0s 655us/sample - loss: 0.1059 - accuracy: 0.9679 - val_loss: 0.6567 - val_accuracy: 0.8500\n",
      "Epoch 33/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.1151 - accuracy: 0.9607 - val_loss: 0.6528 - val_accuracy: 0.8333\n",
      "Epoch 34/100\n",
      "560/560 [==============================] - 0s 634us/sample - loss: 0.0908 - accuracy: 0.9679 - val_loss: 0.6172 - val_accuracy: 0.8500\n",
      "Epoch 35/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0788 - accuracy: 0.9750 - val_loss: 0.7858 - val_accuracy: 0.7833\n",
      "Epoch 36/100\n",
      "560/560 [==============================] - 0s 637us/sample - loss: 0.0873 - accuracy: 0.9732 - val_loss: 0.8790 - val_accuracy: 0.6833\n",
      "Epoch 37/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0870 - accuracy: 0.9696 - val_loss: 0.6617 - val_accuracy: 0.8333\n",
      "Epoch 38/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0867 - accuracy: 0.9714 - val_loss: 0.8417 - val_accuracy: 0.8333\n",
      "Epoch 39/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0680 - accuracy: 0.9768 - val_loss: 0.7361 - val_accuracy: 0.8667\n",
      "Epoch 40/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0797 - accuracy: 0.9732 - val_loss: 0.8301 - val_accuracy: 0.8333\n",
      "Epoch 41/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0566 - accuracy: 0.9875 - val_loss: 0.7760 - val_accuracy: 0.8500\n",
      "Epoch 42/100\n",
      "560/560 [==============================] - 0s 639us/sample - loss: 0.0509 - accuracy: 0.9857 - val_loss: 0.7134 - val_accuracy: 0.8500\n",
      "Epoch 43/100\n",
      "560/560 [==============================] - 0s 636us/sample - loss: 0.0377 - accuracy: 0.9893 - val_loss: 1.1040 - val_accuracy: 0.7667\n",
      "Epoch 44/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0518 - accuracy: 0.9821 - val_loss: 0.6424 - val_accuracy: 0.8000\n",
      "Epoch 45/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0400 - accuracy: 0.9911 - val_loss: 0.7533 - val_accuracy: 0.8500\n",
      "Epoch 46/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0407 - accuracy: 0.9893 - val_loss: 0.9997 - val_accuracy: 0.8000\n",
      "Epoch 47/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0253 - accuracy: 0.9946 - val_loss: 0.8054 - val_accuracy: 0.8667\n",
      "Epoch 48/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0273 - accuracy: 0.9929 - val_loss: 0.9682 - val_accuracy: 0.8333\n",
      "Epoch 49/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0320 - accuracy: 0.9929 - val_loss: 0.8682 - val_accuracy: 0.8667\n",
      "Epoch 50/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0461 - accuracy: 0.9857 - val_loss: 0.6856 - val_accuracy: 0.8333\n",
      "Epoch 51/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0364 - accuracy: 0.9911 - val_loss: 0.7585 - val_accuracy: 0.8667\n",
      "Epoch 52/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0316 - accuracy: 0.9893 - val_loss: 0.7905 - val_accuracy: 0.8333\n",
      "Epoch 53/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0158 - accuracy: 0.9982 - val_loss: 0.8208 - val_accuracy: 0.8500\n",
      "Epoch 54/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0271 - accuracy: 0.9893 - val_loss: 0.8136 - val_accuracy: 0.8667\n",
      "Epoch 55/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0239 - accuracy: 0.9911 - val_loss: 1.0411 - val_accuracy: 0.8500\n",
      "Epoch 56/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0276 - accuracy: 0.9893 - val_loss: 0.9158 - val_accuracy: 0.7667\n",
      "Epoch 57/100\n",
      "560/560 [==============================] - 0s 630us/sample - loss: 0.0233 - accuracy: 0.9946 - val_loss: 1.0975 - val_accuracy: 0.8167\n",
      "Epoch 58/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0219 - accuracy: 0.9893 - val_loss: 1.0286 - val_accuracy: 0.8333\n",
      "Epoch 59/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0335 - accuracy: 0.9875 - val_loss: 0.8941 - val_accuracy: 0.8167\n",
      "Epoch 60/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0270 - accuracy: 0.9893 - val_loss: 0.9775 - val_accuracy: 0.8333\n",
      "Epoch 61/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0126 - accuracy: 0.9982 - val_loss: 0.8844 - val_accuracy: 0.8500\n",
      "Epoch 62/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0365 - accuracy: 0.9875 - val_loss: 0.8818 - val_accuracy: 0.8500\n",
      "Epoch 63/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0126 - accuracy: 0.9964 - val_loss: 0.8641 - val_accuracy: 0.8333\n",
      "Epoch 64/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0232 - accuracy: 0.9929 - val_loss: 1.0328 - val_accuracy: 0.8500\n",
      "Epoch 65/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0110 - accuracy: 1.0000 - val_loss: 1.0617 - val_accuracy: 0.8500\n",
      "Epoch 66/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0176 - accuracy: 0.9964 - val_loss: 0.9585 - val_accuracy: 0.8500\n",
      "Epoch 67/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0259 - accuracy: 0.9911 - val_loss: 1.1278 - val_accuracy: 0.8667\n",
      "Epoch 68/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0173 - accuracy: 0.9929 - val_loss: 1.0505 - val_accuracy: 0.8500\n",
      "Epoch 69/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0249 - accuracy: 0.9964 - val_loss: 1.0193 - val_accuracy: 0.8500\n",
      "Epoch 70/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0422 - accuracy: 0.9893 - val_loss: 1.0280 - val_accuracy: 0.8667\n",
      "Epoch 71/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0265 - accuracy: 0.9946 - val_loss: 1.0278 - val_accuracy: 0.8333\n",
      "Epoch 72/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0244 - accuracy: 0.9911 - val_loss: 1.1455 - val_accuracy: 0.8667\n",
      "Epoch 73/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0203 - accuracy: 0.9946 - val_loss: 1.0028 - val_accuracy: 0.7667\n",
      "Epoch 74/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0210 - accuracy: 0.9964 - val_loss: 1.0002 - val_accuracy: 0.8667\n",
      "Epoch 75/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0145 - accuracy: 0.9946 - val_loss: 0.9576 - val_accuracy: 0.8333\n",
      "Epoch 76/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0195 - accuracy: 0.9911 - val_loss: 0.9737 - val_accuracy: 0.8333\n",
      "Epoch 77/100\n",
      "560/560 [==============================] - 0s 719us/sample - loss: 0.0305 - accuracy: 0.9875 - val_loss: 0.9749 - val_accuracy: 0.8000\n",
      "Epoch 78/100\n",
      "560/560 [==============================] - 0s 635us/sample - loss: 0.0263 - accuracy: 0.9875 - val_loss: 1.2169 - val_accuracy: 0.8167\n",
      "Epoch 79/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0120 - accuracy: 0.9946 - val_loss: 1.1289 - val_accuracy: 0.8333\n",
      "Epoch 80/100\n",
      "560/560 [==============================] - 0s 637us/sample - loss: 0.0092 - accuracy: 1.0000 - val_loss: 1.0046 - val_accuracy: 0.8333\n",
      "Epoch 81/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0130 - accuracy: 0.9946 - val_loss: 1.0671 - val_accuracy: 0.8333\n",
      "Epoch 82/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0086 - accuracy: 1.0000 - val_loss: 1.0507 - val_accuracy: 0.8333\n",
      "Epoch 83/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0036 - accuracy: 1.0000 - val_loss: 1.1542 - val_accuracy: 0.8667\n",
      "Epoch 84/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0123 - accuracy: 0.9964 - val_loss: 1.1238 - val_accuracy: 0.8167\n",
      "Epoch 85/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0137 - accuracy: 0.9964 - val_loss: 1.2414 - val_accuracy: 0.8000\n",
      "Epoch 86/100\n",
      "560/560 [==============================] - 0s 631us/sample - loss: 0.0057 - accuracy: 0.9982 - val_loss: 1.1953 - val_accuracy: 0.8167\n",
      "Epoch 87/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0184 - accuracy: 0.9964 - val_loss: 1.0902 - val_accuracy: 0.8167\n",
      "Epoch 88/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0183 - accuracy: 0.9929 - val_loss: 1.3254 - val_accuracy: 0.8167\n",
      "Epoch 89/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0185 - accuracy: 0.9929 - val_loss: 0.9809 - val_accuracy: 0.8000\n",
      "Epoch 90/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0234 - accuracy: 0.9911 - val_loss: 0.9855 - val_accuracy: 0.8000\n",
      "Epoch 91/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0111 - accuracy: 0.9946 - val_loss: 1.1498 - val_accuracy: 0.8500\n",
      "Epoch 92/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0091 - accuracy: 0.9982 - val_loss: 1.0002 - val_accuracy: 0.8000\n",
      "Epoch 93/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0121 - accuracy: 0.9964 - val_loss: 1.1747 - val_accuracy: 0.8667\n",
      "Epoch 94/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0140 - accuracy: 0.9982 - val_loss: 1.0303 - val_accuracy: 0.8000\n",
      "Epoch 95/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0071 - accuracy: 0.9964 - val_loss: 1.2577 - val_accuracy: 0.8500\n",
      "Epoch 96/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0111 - accuracy: 0.9929 - val_loss: 1.0407 - val_accuracy: 0.7667\n",
      "Epoch 97/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0051 - accuracy: 1.0000 - val_loss: 1.2583 - val_accuracy: 0.8667\n",
      "Epoch 98/100\n",
      "560/560 [==============================] - 0s 633us/sample - loss: 0.0071 - accuracy: 0.9982 - val_loss: 1.1526 - val_accuracy: 0.8333\n",
      "Epoch 99/100\n",
      "560/560 [==============================] - 0s 632us/sample - loss: 0.0037 - accuracy: 0.9982 - val_loss: 1.1147 - val_accuracy: 0.8333\n",
      "Epoch 100/100\n",
      "560/560 [==============================] - 0s 623us/sample - loss: 0.0047 - accuracy: 0.9982 - val_loss: 1.2141 - val_accuracy: 0.8667\n",
      "accuracy for model 10 is 86.66666746139526\n",
      "Training Testing Accuracy: 83.50% (5.40%)\n"
     ]
    }
   ],
   "source": [
    "best_DNN = eval_dnn(tt_vcf, tt_pheno, 10, mlb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(best_DNN, open(\"SCC_DNN_model.pickle.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Holdout accuracy is 89.03225660324097\n"
     ]
    }
   ],
   "source": [
    "bs = ((ho_vcf.shape[0])/40)\n",
    "bs = round(bs)\n",
    "ho_pheno = mlb.transform(ho_pheno)\n",
    "_, accuracy = best_DNN.evaluate(ho_vcf, ho_pheno, batch_size=bs, verbose=0)\n",
    "print(\"Holdout accuracy is \" + str(accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in from file\n",
    "#boy did this get messy lmao\n",
    "my_list = []\n",
    "#import label and one hotencoders\n",
    "le = LabelEncoder()\n",
    "ohe = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
    "imp = SimpleImputer(missing_values='./.', strategy='most_frequent')\n",
    "#reads it in chunks\n",
    "x=0\n",
    "flag = 0\n",
    "#drop half the data for memory problems >:(\n",
    "for chunk in pd.read_csv(\"SCC_filtered_5pcnt.csv\", chunksize=500, index_col=\"Unnamed: 0\"):\n",
    "    x=x+500\n",
    "    chunk = chunk.T\n",
    "    #checks if its the last chunk as Value is the last column\n",
    "    if 'Value' in chunk.columns:\n",
    "        #does the selecting of pheno array for application ML\n",
    "        chunk[\"Value\"] = pd.to_numeric(chunk[\"Value\"], downcast=\"float\")\n",
    "        pheno = chunk[\"Value\"].to_numpy()\n",
    "        #reshapes it so its not a 1D array\n",
    "        print(pheno.shape)\n",
    "        pheno = np.reshape(pheno,(len(pheno),1))\n",
    "        print(pheno.shape)\n",
    "        chunk = chunk.drop(columns=['Value'])\n",
    "    #applies label and OHE to each chunk, won't include Value column as it'd been dropped\n",
    "    headers = chunk.columns\n",
    "    row_idx = chunk.index\n",
    "    chunk = imp.fit_transform(chunk) #SHOULD TURN ./. into the most common for each column\n",
    "    chunk = pd.DataFrame(data = chunk, index = row_idx, columns = headers)\n",
    "    chunk = chunk.apply(lambda col: le.fit_transform(col))\n",
    "    chunk = ohe.fit_transform(chunk)\n",
    "    print(chunk.shape)\n",
    "    if(flag==0):\n",
    "        my_list.append(chunk)\n",
    "        print(str(x))\n",
    "        flag = 1\n",
    "    else:\n",
    "        flag = 0\n",
    "#concats the chunks back in to prevent pandas having a heart attack\n",
    "#print(pheno.shape)\n",
    "#my_list.append(pheno)\n",
    "vcf = np.concatenate(my_list, axis = 1)\n",
    "print(vcf)\n",
    "print(pheno)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converts the values into a onehot encoded array of each output\n",
    "#eg. a range of 0,1,2,3 for SCC is converted into 1,0,0,0  0,1,0,0  0,0,1,0  0,0,0,1\n",
    "mlb = MultiLabelBinarizer()\n",
    "pheno = mlb.fit_transform(pheno)\n",
    "print(pheno)\n",
    "print(pheno.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#divide up the training and testing data here\n",
    "X_train, X_test, y_train, y_test = train_test_split(vcf, pheno, test_size=0.2, random_state=27022013)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#My own DNN model based upon paper\n",
    "#del model #incase its stored a previous model\n",
    "#del history #for redoing shit\n",
    "\n",
    "#do batch size as 64\n",
    "#reduce the inputs by half when you read it in\n",
    "#add XGboost and RF to the one notebook\n",
    "def DNN_model():\n",
    "    model = Sequential()\n",
    "\n",
    "    #add first input layer, with no normalization\n",
    "    model.add(Dense(100, input_dim = X_train.shape[1], kernel_initializer='glorot_normal', activity_regularizer=l1(0.0001)))\n",
    "    model.add(Activation('relu'))\n",
    "\n",
    "    #add 21 hidden layers with l2 regularization and batch Normalization before activation\n",
    "    i = 0\n",
    "    while(i < 21):\n",
    "            model.add(Dense(50, kernel_initializer='glorot_normal', activity_regularizer=l2(0.0001)))\n",
    "            model.add(BatchNormalization())\n",
    "            model.add(Activation('relu'))\n",
    "            i = i + 1\n",
    "\n",
    "    #add output layer\n",
    "    model.add(Dense(4, activation='softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=250, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure()\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, accuracy = model.evaluate(X_test, y_test, batch_size=64, verbose=0)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vcf = vcf.reshape(vcf.shape[0], vcf.shape[1])\n",
    "pheno = pheno.reshape(pheno.shape[0], pheno.shape[1])\n",
    "estimator = KerasClassifier(build_fn=DNN_model, epochs=200, batch_size=64, verbose=1)\n",
    "kfold = KFold(n_splits=10, shuffle=True)\n",
    "results = cross_val_score(estimator, vcf, pheno, cv=kfold)\n",
    "print(\"Baseline: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}