Unet_Faults_CV.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\n",
    "from skimage.transform import resize\n",
    "\n",
    "import keras\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input, BatchNormalization, Activation, Dense, Dropout\n",
    "from keras.layers.core import Lambda, RepeatVector, Reshape\n",
    "from keras.layers.convolutional import Conv2D, Conv2DTranspose\n",
    "from keras.layers.pooling import MaxPooling2D, GlobalMaxPool2D\n",
    "from keras.layers.merge import concatenate, add\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, Callback\n",
    "from keras.optimizers import Adam\n",
    "from keras import regularizers\n",
    "from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n",
    "\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def f1(y_true, y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        \"\"\"Recall metric.\n",
    "\n",
    "        Only computes a batch-wise average of recall.\n",
    "\n",
    "        Computes the recall, a metric for multi-label classification of\n",
    "        how many relevant items are selected.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        \"\"\"Precision metric.\n",
    "\n",
    "        Only computes a batch-wise average of precision.\n",
    "\n",
    "        Computes the precision, a metric for multi-label classification of\n",
    "        how many selected items are relevant.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def dice_coef(y_true, y_pred, smooth=1):\n",
    "    \"\"\"\n",
    "    Dice = (2*|X & Y|)/ (|X|+ |Y|)\n",
    "         =  2*sum(|A*B|)/(sum(A^2)+sum(B^2))\n",
    "    ref: https://arxiv.org/pdf/1606.04797v1.pdf\n",
    "    \"\"\"\n",
    "    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n",
    "    return (2. * intersection + smooth) / (K.sum(K.square(y_true),-1) + K.sum(K.square(y_pred),-1) + smooth)\n",
    "\n",
    "def dice_coef_loss(y_true, y_pred):\n",
    "    return 1-dice_coef(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jaccard_distance_loss(y_true, y_pred, smooth=100):\n",
    "    \"\"\"\n",
    "    Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|)\n",
    "            = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|))\n",
    "    \n",
    "    The jaccard distance loss is usefull for unbalanced datasets. This has been\n",
    "    shifted so it converges on 0 and is smoothed to avoid exploding or disapearing\n",
    "    gradient.\n",
    "    \n",
    "    Ref: https://en.wikipedia.org/wiki/Jaccard_index\n",
    "    \n",
    "    @url: https://gist.github.com/wassname/f1452b748efcbeb4cb9b1d059dce6f96\n",
    "    @author: wassname\n",
    "    \"\"\"\n",
    "    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n",
    "    sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1)\n",
    "    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n",
    "    return (1 - jac) * smooth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_roc_metrics(y_real, y_predict):\n",
    "\n",
    "    c_matrix = confusion_matrix(y_real.ravel(), y_predict.ravel())\n",
    "    f1 = f1_score(y_real.ravel(), y_predict.ravel())\n",
    "    recall = recall_score(y_real.ravel(), y_predict.ravel())\n",
    "    precision = precision_score(y_real.ravel(), y_predict.ravel())\n",
    "    print(\"Confusion matrix:\")\n",
    "    print(c_matrix)\n",
    "    print(\"F1 score: {:.4f}\".format(f1))\n",
    "    print(\"Recall score: {:.4f}\".format(recall))\n",
    "    print(\"Precision score: {:.4f}\".format(precision))\n",
    "    \n",
    "    return c_matrix, f1, recall, precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "source_string =  'vertical' #'horizontal' #\n",
    "receiver_string =  'combined' #'combined',  'das' 'geophone'\n",
    "##foldstring = 2 #0, 1, or 2\n",
    "##values4random_state=[1234, 69, 753]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if source_string == 'horizontal':\n",
    "    data = np.load(r'./horizontal_data/h_'+receiver_string+'_data.npy') \n",
    "    labels = np.load(r'./horizontal_data/h_fault_labels.npy')\n",
    "else:\n",
    "    data = np.load(r'./vertical_data/v_'+receiver_string+'_data.npy')\n",
    "    labels = np.load(r'./vertical_data/v_fault_labels.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nx = 256\n",
    "ny = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda/envs/py35/lib/python3.5/site-packages/skimage/transform/_warps.py:84: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.\n",
      "  warn(\"The default mode, 'constant', will be changed to 'reflect' in \"\n"
     ]
    }
   ],
   "source": [
    "resized_data = np.zeros((len(data), nx, ny, 1))\n",
    "resized_labels = np.zeros((len(data), nx, ny, 2))\n",
    "for i in range(len(resized_data)):\n",
    "    resized_data[i,:,:,0] = resize(data[i,:,:,0], (nx, ny))\n",
    "    resized_labels[i,:,:,1] = resize(labels[i,:,:,0], (nx, ny))\n",
    "    resized_labels[i,:,:,0] = resize(labels[i,:,:,1], (nx, ny))\n",
    "resized_labels[resized_labels <= 0.5] = 0\n",
    "resized_labels[resized_labels > 0.5] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Standardization across entire dataset\n",
    "data_mean = np.mean(resized_data)\n",
    "data_std = np.std(resized_data)\n",
    "data_scaled = (resized_data-data_mean)/data_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = data_scaled\n",
    "y = resized_labels[:,:,:,1]\n",
    "y = np.expand_dims(y, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d_block(input_tensor, n_filters, kernel_size=3, l2_lambda=5e-5, batch_norm=True):\n",
    "    \"\"\"\n",
    "    A single convolution block for Unet.\n",
    "    Convolution -> Batch Normalization -> Activation -> Repeat\n",
    "    \"\"\"\n",
    "    # first layer\n",
    "    conv_0 = Conv2D(filters=n_filters, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n",
    "               padding=\"same\", kernel_regularizer=regularizers.l2(l2_lambda))(input_tensor)\n",
    "    if batch_norm:\n",
    "        conv_0 = BatchNormalization()(conv_0)\n",
    "    conv_0 = Activation(\"relu\")(conv_0)\n",
    "    # second layer\n",
    "    conv_1 = Conv2D(filters=n_filters, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n",
    "               padding=\"same\", kernel_regularizer=regularizers.l2(l2_lambda))(conv_0)\n",
    "    if batch_norm:\n",
    "        conv_1 = BatchNormalization()(conv_1)\n",
    "    conv_1 = Activation(\"relu\")(conv_1)\n",
    "    return conv_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_unet(input_image, n_filters=16, dropout=0.5, batch_norm=True):\n",
    "    \"\"\"\n",
    "    Create U-net similar to that described by Ronneberger et al. 2015, with some changes:\n",
    "    Output size will equal input size for all convolutioin operations\n",
    "    Dropout is applied after every max pooling or upconvolution operation.\n",
    "    L2 regularization is applied after every convolution\n",
    "    \"\"\"\n",
    "    # downsampling layers\n",
    "    c_d_0 = conv2d_block(input_image, n_filters=n_filters, kernel_size=3, batch_norm=batch_norm)\n",
    "    p_d_0 = MaxPooling2D((2,2))(c_d_0)\n",
    "    d_d_0 = Dropout(dropout*0.5)(p_d_0)\n",
    "\n",
    "    c_d_1 = conv2d_block(d_d_0, n_filters=n_filters*2, kernel_size=3, batch_norm=batch_norm)\n",
    "    p_d_1 = MaxPooling2D((2,2))(c_d_1)\n",
    "    d_d_1 = Dropout(dropout)(p_d_1)\n",
    "\n",
    "    c_d_2 = conv2d_block(d_d_1, n_filters=n_filters*4, kernel_size=3, batch_norm=batch_norm)\n",
    "    p_d_2 = MaxPooling2D((2,2))(c_d_2)\n",
    "    d_d_2 = Dropout(dropout)(p_d_2)\n",
    "\n",
    "    c_d_3 = conv2d_block(d_d_2, n_filters=n_filters*8, kernel_size=3, batch_norm=batch_norm)\n",
    "    p_d_3= MaxPooling2D(pool_size=(2,2))(c_d_3)\n",
    "    d_d_3 = Dropout(dropout)(p_d_3)\n",
    "    \n",
    "    c_d_4 = conv2d_block(d_d_3, n_filters=n_filters*16, kernel_size=3, batch_norm=batch_norm)\n",
    "    \n",
    "    # upsampling layers\n",
    "    u_u_3 = Conv2DTranspose(n_filters*8, (3,3), strides=(2,2), padding='same')(c_d_4)\n",
    "    # skip layer 3\n",
    "    cc_u_3 = concatenate([u_u_3, c_d_3])\n",
    "    d_u_3 = Dropout(dropout)(cc_u_3)\n",
    "    c_u_3 = conv2d_block(d_u_3, n_filters=n_filters*8, kernel_size=3, batch_norm=batch_norm)\n",
    "\n",
    "    u_u_2 = Conv2DTranspose(n_filters*4, (3,3), strides=(2,2), padding='same')(c_u_3)\n",
    "    cc_u_2 = concatenate([u_u_2, c_d_2])\n",
    "    # skip layer 2\n",
    "    d_u_2 = Dropout(dropout)(cc_u_2)\n",
    "    c_u_2 = conv2d_block(d_u_2, n_filters=n_filters*4, kernel_size=3, batch_norm=batch_norm)\n",
    "\n",
    "    u_u_1 = Conv2DTranspose(n_filters*2, (3,3), strides=(2,2), padding='same')(c_u_2)\n",
    "    # skip layer 1\n",
    "    cc_u_1 = concatenate([u_u_1, c_d_1])\n",
    "    d_u_1 = Dropout(dropout)(cc_u_1)\n",
    "    c_u_1 = conv2d_block(d_u_1, n_filters=n_filters*2, kernel_size=3, batch_norm=batch_norm)\n",
    "\n",
    "    u_u_0 = Conv2DTranspose(n_filters, (3,3), strides=(2,2), padding='same')(c_u_1)\n",
    "    # skip layer 0\n",
    "    cc_u_0 = concatenate([u_u_0, c_d_0], axis=3)\n",
    "    d_u_0 = Dropout(dropout)(cc_u_0)\n",
    "    c_u_0 = conv2d_block(d_u_0, n_filters=n_filters, kernel_size=3, batch_norm=batch_norm)\n",
    "    \n",
    "    output_segmentation = Conv2D(1, (1, 1), activation='sigmoid')(c_u_0)\n",
    "    model = Model(inputs=[input_img], outputs=[output_segmentation])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# lists for holding statistics from CV\n",
    "eval_stats = []\n",
    "c_matrices = []\n",
    "f1_stats = []\n",
    "recall_stats = []\n",
    "precision_stats = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reshaping y for kfolds.split (original shape still used for training)\n",
    "y_reshaped = y.reshape((len(y), y.shape[1]*y.shape[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform K fold split for CV\n",
    "Step 2. Change the ModelCheckpoint argument to name your models generated from each fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "?KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfolds = KFold(n_splits=3, shuffle=True, random_state=69) #random_state was not set for 1st CV of Horizontal DAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/v-model-combined-fold\n"
     ]
    }
   ],
   "source": [
    "# NameCheckpoint = 'models/'+source_string[0]+'-model-'+receiver_string+'-dice-fold-'#+str(foldstring)+'.h5'\n",
    "NameCheckpoint = 'models/'+source_string[0]+'-model-'+receiver_string+'-fold'#+str(foldstring)+'.h5'\n",
    "print(NameCheckpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time 2.193450927734375e-05\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "t = time.time()\n",
    "# do stuff\n",
    "elapsed = time.time() - t\n",
    "print('elapsed time',elapsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Load previous model?\n",
    "LoadPrevious = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for fold 0\n",
      "size of LOADED weights: (118,)\n",
      "[[[-0.28594556  0.06087189 -0.35679764  0.20312005 -0.01559232\n",
      "    0.06216661  0.13975424  0.11081334  0.10477987  0.07890682\n",
      "    0.29468048 -0.015343   -0.09282309  0.0760137   0.04808076\n",
      "    0.26520285]]\n",
      "\n",
      " [[ 0.0941376   0.16903515 -0.00741779  0.20390736 -0.26758742\n",
      "    0.04122241  0.05460102  0.25423115 -0.21888216  0.02384838\n",
      "   -0.01416525  0.04251644  0.642405   -0.00795194  0.06827527\n",
      "    0.02696223]]\n",
      "\n",
      " [[ 0.07195768 -0.32874587  0.30954933 -0.06390805  0.24363063\n",
      "   -0.02802558  0.00417702 -0.32010344  0.22017239 -0.00659807\n",
      "    0.3808607  -0.14013039  0.00654059 -0.08051324  0.05770043\n",
      "    0.04404148]]]\n",
      "Train on 122 samples, validate on 61 samples\n",
      "Epoch 1/100\n",
      "122/122 [==============================] - 94s 768ms/step - loss: 0.0054 - acc: 0.9925 - f1: 0.8616 - val_loss: 0.0048 - val_acc: 0.9930 - val_f1: 0.8496\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.00485, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 2/100\n",
      "122/122 [==============================] - 91s 742ms/step - loss: 0.0045 - acc: 0.9935 - f1: 0.8777 - val_loss: 0.0052 - val_acc: 0.9924 - val_f1: 0.8267\n",
      "\n",
      "Epoch 00002: val_loss did not improve\n",
      "Epoch 3/100\n",
      "122/122 [==============================] - 90s 738ms/step - loss: 0.0052 - acc: 0.9931 - f1: 0.8683 - val_loss: 0.0166 - val_acc: 0.9735 - val_f1: 4.5025e-05\n",
      "\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 4/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0059 - acc: 0.9926 - f1: 0.8618 - val_loss: 0.0056 - val_acc: 0.9918 - val_f1: 0.8097\n",
      "\n",
      "Epoch 00004: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0043 - acc: 0.9946 - f1: 0.8997 - val_loss: 0.0047 - val_acc: 0.9934 - val_f1: 0.8486\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.00485 to 0.00474, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 6/100\n",
      "122/122 [==============================] - 91s 747ms/step - loss: 0.0039 - acc: 0.9952 - f1: 0.9100 - val_loss: 0.0045 - val_acc: 0.9937 - val_f1: 0.8542\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.00474 to 0.00448, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 7/100\n",
      "122/122 [==============================] - 91s 745ms/step - loss: 0.0037 - acc: 0.9954 - f1: 0.9149 - val_loss: 0.0044 - val_acc: 0.9937 - val_f1: 0.8576\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.00448 to 0.00437, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 8/100\n",
      "122/122 [==============================] - 92s 757ms/step - loss: 0.0035 - acc: 0.9956 - f1: 0.9177 - val_loss: 0.0042 - val_acc: 0.9940 - val_f1: 0.8628\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.00437 to 0.00419, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 9/100\n",
      "122/122 [==============================] - 92s 755ms/step - loss: 0.0034 - acc: 0.9957 - f1: 0.9204 - val_loss: 0.0041 - val_acc: 0.9939 - val_f1: 0.8594\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.00419 to 0.00415, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 10/100\n",
      "122/122 [==============================] - 91s 750ms/step - loss: 0.0033 - acc: 0.9959 - f1: 0.9230 - val_loss: 0.0041 - val_acc: 0.9940 - val_f1: 0.8625\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.00415 to 0.00408, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 11/100\n",
      "122/122 [==============================] - 93s 766ms/step - loss: 0.0032 - acc: 0.9959 - f1: 0.9236 - val_loss: 0.0040 - val_acc: 0.9940 - val_f1: 0.8591\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.00408 to 0.00401, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 12/100\n",
      "122/122 [==============================] - 89s 729ms/step - loss: 0.0031 - acc: 0.9960 - f1: 0.9262 - val_loss: 0.0040 - val_acc: 0.9939 - val_f1: 0.8577\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.00401 to 0.00397, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 13/100\n",
      "122/122 [==============================] - 88s 725ms/step - loss: 0.0030 - acc: 0.9962 - f1: 0.9287 - val_loss: 0.0038 - val_acc: 0.9943 - val_f1: 0.8713\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.00397 to 0.00382, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 14/100\n",
      "122/122 [==============================] - 89s 726ms/step - loss: 0.0029 - acc: 0.9962 - f1: 0.9296 - val_loss: 0.0037 - val_acc: 0.9944 - val_f1: 0.8697\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.00382 to 0.00372, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 15/100\n",
      "122/122 [==============================] - 89s 726ms/step - loss: 0.0029 - acc: 0.9963 - f1: 0.9307 - val_loss: 0.0037 - val_acc: 0.9945 - val_f1: 0.8763\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.00372 to 0.00369, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 16/100\n",
      "122/122 [==============================] - 88s 722ms/step - loss: 0.0028 - acc: 0.9964 - f1: 0.9322 - val_loss: 0.0036 - val_acc: 0.9946 - val_f1: 0.8779\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.00369 to 0.00357, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 17/100\n",
      "122/122 [==============================] - 88s 721ms/step - loss: 0.0027 - acc: 0.9965 - f1: 0.9354 - val_loss: 0.0036 - val_acc: 0.9945 - val_f1: 0.8751\n",
      "\n",
      "Epoch 00017: val_loss did not improve\n",
      "Epoch 18/100\n",
      "122/122 [==============================] - 88s 719ms/step - loss: 0.0027 - acc: 0.9966 - f1: 0.9359 - val_loss: 0.0036 - val_acc: 0.9944 - val_f1: 0.8723\n",
      "\n",
      "Epoch 00018: val_loss did not improve\n",
      "Epoch 19/100\n",
      "122/122 [==============================] - 91s 743ms/step - loss: 0.0026 - acc: 0.9967 - f1: 0.9383 - val_loss: 0.0036 - val_acc: 0.9943 - val_f1: 0.8691\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/100\n",
      "122/122 [==============================] - 95s 775ms/step - loss: 0.0025 - acc: 0.9970 - f1: 0.9430 - val_loss: 0.0034 - val_acc: 0.9949 - val_f1: 0.8849\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.00357 to 0.00338, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 21/100\n",
      "122/122 [==============================] - 91s 749ms/step - loss: 0.0025 - acc: 0.9970 - f1: 0.9441 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8873\n",
      "\n",
      "Epoch 00021: val_loss improved from 0.00338 to 0.00334, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 22/100\n",
      "122/122 [==============================] - 92s 751ms/step - loss: 0.0024 - acc: 0.9971 - f1: 0.9450 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8896\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.00334 to 0.00332, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 23/100\n",
      "122/122 [==============================] - 89s 727ms/step - loss: 0.0024 - acc: 0.9971 - f1: 0.9458 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8897\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1e-05.\n",
      "\n",
      "Epoch 00023: val_loss improved from 0.00332 to 0.00331, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 24/100\n",
      "122/122 [==============================] - 89s 728ms/step - loss: 0.0024 - acc: 0.9971 - f1: 0.9461 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8882\n",
      "\n",
      "Epoch 00024: val_loss did not improve\n",
      "Epoch 25/100\n",
      "122/122 [==============================] - 88s 723ms/step - loss: 0.0024 - acc: 0.9971 - f1: 0.9463 - val_loss: 0.0033 - val_acc: 0.9949 - val_f1: 0.8857\n",
      "\n",
      "Epoch 00025: val_loss did not improve\n",
      "Epoch 26/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0024 - acc: 0.9972 - f1: 0.9466 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8890\n",
      "\n",
      "Epoch 00026: val_loss improved from 0.00331 to 0.00330, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 27/100\n",
      "122/122 [==============================] - 91s 748ms/step - loss: 0.0024 - acc: 0.9972 - f1: 0.9468 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8865\n",
      "\n",
      "Epoch 00027: val_loss did not improve\n",
      "Epoch 28/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0024 - acc: 0.9972 - f1: 0.9473 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8863\n",
      "\n",
      "Epoch 00028: val_loss did not improve\n",
      "Epoch 29/100\n",
      "122/122 [==============================] - 93s 763ms/step - loss: 0.0023 - acc: 0.9972 - f1: 0.9479 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8892\n",
      "\n",
      "Epoch 00029: val_loss improved from 0.00330 to 0.00329, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 30/100\n",
      "122/122 [==============================] - 96s 787ms/step - loss: 0.0023 - acc: 0.9973 - f1: 0.9487 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8896\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.00329 to 0.00328, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 31/100\n",
      "122/122 [==============================] - 94s 772ms/step - loss: 0.0023 - acc: 0.9973 - f1: 0.9486 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8891\n",
      "\n",
      "Epoch 00031: val_loss did not improve\n",
      "Epoch 32/100\n",
      "122/122 [==============================] - 95s 782ms/step - loss: 0.0023 - acc: 0.9973 - f1: 0.9493 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8891\n",
      "\n",
      "Epoch 00032: val_loss improved from 0.00328 to 0.00327, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 33/100\n",
      "122/122 [==============================] - 95s 778ms/step - loss: 0.0023 - acc: 0.9973 - f1: 0.9495 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8888\n",
      "\n",
      "Epoch 00033: val_loss did not improve\n",
      "Epoch 34/100\n",
      "122/122 [==============================] - 95s 778ms/step - loss: 0.0023 - acc: 0.9974 - f1: 0.9499 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8890\n",
      "\n",
      "Epoch 00034: val_loss improved from 0.00327 to 0.00327, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 35/100\n",
      "122/122 [==============================] - 100s 817ms/step - loss: 0.0023 - acc: 0.9974 - f1: 0.9500 - val_loss: 0.0033 - val_acc: 0.9951 - val_f1: 0.8909\n",
      "\n",
      "Epoch 00035: val_loss improved from 0.00327 to 0.00326, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 36/100\n",
      "122/122 [==============================] - 98s 802ms/step - loss: 0.0023 - acc: 0.9974 - f1: 0.9505 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8911\n",
      "\n",
      "Epoch 00036: val_loss improved from 0.00326 to 0.00325, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 37/100\n",
      "122/122 [==============================] - 91s 746ms/step - loss: 0.0023 - acc: 0.9974 - f1: 0.9504 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8884\n",
      "\n",
      "Epoch 00037: val_loss did not improve\n",
      "Epoch 38/100\n",
      "122/122 [==============================] - 89s 731ms/step - loss: 0.0022 - acc: 0.9974 - f1: 0.9509 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8892\n",
      "\n",
      "Epoch 00038: val_loss did not improve\n",
      "Epoch 39/100\n",
      "122/122 [==============================] - 90s 738ms/step - loss: 0.0022 - acc: 0.9974 - f1: 0.9517 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8899\n",
      "\n",
      "Epoch 00039: val_loss did not improve\n",
      "Epoch 40/100\n",
      "122/122 [==============================] - 91s 745ms/step - loss: 0.0022 - acc: 0.9975 - f1: 0.9524 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8904\n",
      "\n",
      "Epoch 00040: val_loss improved from 0.00325 to 0.00324, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 41/100\n",
      "122/122 [==============================] - 92s 756ms/step - loss: 0.0022 - acc: 0.9975 - f1: 0.9520 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8904\n",
      "\n",
      "Epoch 00041: val_loss improved from 0.00324 to 0.00323, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 42/100\n",
      "122/122 [==============================] - 92s 757ms/step - loss: 0.0022 - acc: 0.9975 - f1: 0.9523 - val_loss: 0.0033 - val_acc: 0.9951 - val_f1: 0.8898\n",
      "\n",
      "Epoch 00042: val_loss did not improve\n",
      "Epoch 43/100\n",
      "122/122 [==============================] - 94s 774ms/step - loss: 0.0022 - acc: 0.9975 - f1: 0.9525 - val_loss: 0.0033 - val_acc: 0.9951 - val_f1: 0.8899\n",
      "\n",
      "Epoch 00043: val_loss did not improve\n",
      "Epoch 44/100\n",
      "122/122 [==============================] - 97s 791ms/step - loss: 0.0022 - acc: 0.9975 - f1: 0.9535 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8882\n",
      "\n",
      "Epoch 00044: val_loss did not improve\n",
      "Epoch 45/100\n",
      "122/122 [==============================] - 97s 795ms/step - loss: 0.0022 - acc: 0.9976 - f1: 0.9542 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8906\n",
      "\n",
      "Epoch 00045: val_loss improved from 0.00323 to 0.00321, saving model to models/v-model-combined-fold-0_100epoch.h5\n",
      "Epoch 46/100\n",
      "122/122 [==============================] - 102s 837ms/step - loss: 0.0022 - acc: 0.9976 - f1: 0.9541 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8885\n",
      "\n",
      "Epoch 00046: val_loss did not improve\n",
      "Epoch 47/100\n",
      "122/122 [==============================] - 102s 840ms/step - loss: 0.0022 - acc: 0.9976 - f1: 0.9540 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8889\n",
      "\n",
      "Epoch 00047: val_loss did not improve\n",
      "Epoch 48/100\n",
      "122/122 [==============================] - 100s 818ms/step - loss: 0.0021 - acc: 0.9976 - f1: 0.9548 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8908\n",
      "\n",
      "Epoch 00048: val_loss did not improve\n",
      "Epoch 49/100\n",
      "122/122 [==============================] - 99s 808ms/step - loss: 0.0021 - acc: 0.9976 - f1: 0.9549 - val_loss: 0.0032 - val_acc: 0.9951 - val_f1: 0.8895\n",
      "\n",
      "Epoch 00049: val_loss did not improve\n",
      "Epoch 50/100\n",
      "122/122 [==============================] - 94s 773ms/step - loss: 0.0021 - acc: 0.9976 - f1: 0.9552 - val_loss: 0.0033 - val_acc: 0.9950 - val_f1: 0.8841\n",
      "\n",
      "Epoch 00050: val_loss did not improve\n",
      "Epoch 00050: early stopping\n",
      "61/61 [==============================] - 14s 226ms/step\n",
      "61/61 [==============================] - 14s 228ms/step\n",
      "Training for fold 1\n",
      "size of LOADED weights: (118,)\n",
      "[[[ 0.00455231  0.00195708 -0.2578603  -0.3622147  -0.03581912\n",
      "    0.29927006 -0.5459097   0.24019246 -0.20942664 -0.14914307\n",
      "   -0.28068656  0.06257665  0.17735998 -0.46848494  0.3150095\n",
      "    0.40041083]]\n",
      "\n",
      " [[ 0.16862215 -0.09499117  0.11719537 -0.00976563  0.18048169\n",
      "    0.12485053 -0.0980108  -0.20131174 -0.37495095  0.00508002\n",
      "   -0.24018162  0.00896181  0.15926473  0.02300513  0.14716548\n",
      "   -0.5749834 ]]\n",
      "\n",
      " [[ 0.05621616 -0.07896301 -0.36068487 -0.13513668  0.11474516\n",
      "    0.04375611  0.18313701  0.21418975  0.15188804 -0.15332636\n",
      "   -0.3188908  -0.08362757  0.02249885 -0.01756058  0.10263587\n",
      "   -0.1834404 ]]]\n",
      "Train on 122 samples, validate on 61 samples\n",
      "Epoch 1/100\n",
      "122/122 [==============================] - 104s 853ms/step - loss: 0.0056 - acc: 0.9925 - f1: 0.8542 - val_loss: 0.0067 - val_acc: 0.9886 - val_f1: 0.7616\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.00669, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 2/100\n",
      "122/122 [==============================] - 97s 794ms/step - loss: 0.0046 - acc: 0.9936 - f1: 0.8755 - val_loss: 0.0057 - val_acc: 0.9903 - val_f1: 0.7930\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.00669 to 0.00573, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 3/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0046 - acc: 0.9935 - f1: 0.8725 - val_loss: 0.0165 - val_acc: 0.9714 - val_f1: 0.0056\n",
      "\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 4/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0050 - acc: 0.9934 - f1: 0.8688 - val_loss: 0.0060 - val_acc: 0.9901 - val_f1: 0.7936\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "122/122 [==============================] - 94s 772ms/step - loss: 0.0045 - acc: 0.9941 - f1: 0.8842 - val_loss: 0.0056 - val_acc: 0.9910 - val_f1: 0.8201\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.00573 to 0.00561, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 6/100\n",
      "122/122 [==============================] - 95s 780ms/step - loss: 0.0046 - acc: 0.9934 - f1: 0.8705 - val_loss: 0.0052 - val_acc: 0.9919 - val_f1: 0.8486\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.00561 to 0.00522, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 7/100\n",
      "122/122 [==============================] - 92s 752ms/step - loss: 0.0041 - acc: 0.9942 - f1: 0.8867 - val_loss: 0.0055 - val_acc: 0.9907 - val_f1: 0.8062\n",
      "\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 8/100\n",
      "122/122 [==============================] - 91s 746ms/step - loss: 0.0049 - acc: 0.9938 - f1: 0.8779 - val_loss: 0.0054 - val_acc: 0.9923 - val_f1: 0.8550\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/100\n",
      "122/122 [==============================] - 91s 743ms/step - loss: 0.0047 - acc: 0.9938 - f1: 0.8803 - val_loss: 0.0070 - val_acc: 0.9892 - val_f1: 0.7688\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "\n",
      "Epoch 00009: val_loss did not improve\n",
      "Epoch 10/100\n",
      "122/122 [==============================] - 91s 746ms/step - loss: 0.0042 - acc: 0.9950 - f1: 0.9007 - val_loss: 0.0045 - val_acc: 0.9936 - val_f1: 0.8809\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.00522 to 0.00449, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 11/100\n",
      "122/122 [==============================] - 91s 748ms/step - loss: 0.0035 - acc: 0.9958 - f1: 0.9178 - val_loss: 0.0041 - val_acc: 0.9939 - val_f1: 0.8876\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.00449 to 0.00413, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 12/100\n",
      "122/122 [==============================] - 93s 765ms/step - loss: 0.0032 - acc: 0.9960 - f1: 0.9216 - val_loss: 0.0041 - val_acc: 0.9938 - val_f1: 0.8828\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.00413 to 0.00407, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 13/100\n",
      "122/122 [==============================] - 91s 744ms/step - loss: 0.0031 - acc: 0.9961 - f1: 0.9241 - val_loss: 0.0039 - val_acc: 0.9940 - val_f1: 0.8884\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.00407 to 0.00389, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 14/100\n",
      "122/122 [==============================] - 90s 742ms/step - loss: 0.0030 - acc: 0.9962 - f1: 0.9262 - val_loss: 0.0038 - val_acc: 0.9941 - val_f1: 0.8908\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.00389 to 0.00379, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 15/100\n",
      "122/122 [==============================] - 90s 739ms/step - loss: 0.0029 - acc: 0.9963 - f1: 0.9279 - val_loss: 0.0043 - val_acc: 0.9930 - val_f1: 0.8671\n",
      "\n",
      "Epoch 00015: val_loss did not improve\n",
      "Epoch 16/100\n",
      "122/122 [==============================] - 89s 734ms/step - loss: 0.0028 - acc: 0.9964 - f1: 0.9304 - val_loss: 0.0037 - val_acc: 0.9941 - val_f1: 0.8892\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.00379 to 0.00373, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 17/100\n",
      "122/122 [==============================] - 89s 732ms/step - loss: 0.0027 - acc: 0.9964 - f1: 0.9301 - val_loss: 0.0036 - val_acc: 0.9943 - val_f1: 0.8947\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.00373 to 0.00361, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 18/100\n",
      "122/122 [==============================] - 89s 730ms/step - loss: 0.0026 - acc: 0.9966 - f1: 0.9328 - val_loss: 0.0036 - val_acc: 0.9942 - val_f1: 0.8914\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.00361 to 0.00357, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 19/100\n",
      "122/122 [==============================] - 90s 741ms/step - loss: 0.0026 - acc: 0.9967 - f1: 0.9345 - val_loss: 0.0036 - val_acc: 0.9941 - val_f1: 0.8903\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/100\n",
      "122/122 [==============================] - 90s 736ms/step - loss: 0.0025 - acc: 0.9967 - f1: 0.9356 - val_loss: 0.0035 - val_acc: 0.9943 - val_f1: 0.8938\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.00357 to 0.00348, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 21/100\n",
      "122/122 [==============================] - 90s 737ms/step - loss: 0.0024 - acc: 0.9968 - f1: 0.9376 - val_loss: 0.0034 - val_acc: 0.9945 - val_f1: 0.8995\n",
      "\n",
      "Epoch 00021: val_loss improved from 0.00348 to 0.00338, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 22/100\n",
      "122/122 [==============================] - 90s 741ms/step - loss: 0.0024 - acc: 0.9969 - f1: 0.9382 - val_loss: 0.0035 - val_acc: 0.9942 - val_f1: 0.8926\n",
      "\n",
      "Epoch 00022: val_loss did not improve\n",
      "Epoch 23/100\n",
      "122/122 [==============================] - 90s 738ms/step - loss: 0.0023 - acc: 0.9969 - f1: 0.9393 - val_loss: 0.0035 - val_acc: 0.9942 - val_f1: 0.8912\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "\n",
      "Epoch 00023: val_loss did not improve\n",
      "Epoch 24/100\n",
      "122/122 [==============================] - 90s 738ms/step - loss: 0.0022 - acc: 0.9972 - f1: 0.9449 - val_loss: 0.0033 - val_acc: 0.9947 - val_f1: 0.9022\n",
      "\n",
      "Epoch 00024: val_loss improved from 0.00338 to 0.00325, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 25/100\n",
      "122/122 [==============================] - 90s 742ms/step - loss: 0.0022 - acc: 0.9973 - f1: 0.9462 - val_loss: 0.0032 - val_acc: 0.9948 - val_f1: 0.9058\n",
      "\n",
      "Epoch 00025: val_loss improved from 0.00325 to 0.00321, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 26/100\n",
      "122/122 [==============================] - 90s 739ms/step - loss: 0.0022 - acc: 0.9973 - f1: 0.9472 - val_loss: 0.0032 - val_acc: 0.9948 - val_f1: 0.9070\n",
      "\n",
      "Epoch 00026: val_loss improved from 0.00321 to 0.00318, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 27/100\n",
      "122/122 [==============================] - 90s 735ms/step - loss: 0.0021 - acc: 0.9973 - f1: 0.9479 - val_loss: 0.0032 - val_acc: 0.9948 - val_f1: 0.9075\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1e-05.\n",
      "\n",
      "Epoch 00027: val_loss did not improve\n",
      "Epoch 28/100\n",
      "122/122 [==============================] - 90s 740ms/step - loss: 0.0021 - acc: 0.9973 - f1: 0.9481 - val_loss: 0.0032 - val_acc: 0.9948 - val_f1: 0.9064\n",
      "\n",
      "Epoch 00028: val_loss improved from 0.00318 to 0.00317, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 29/100\n",
      "122/122 [==============================] - 90s 739ms/step - loss: 0.0021 - acc: 0.9974 - f1: 0.9487 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9079\n",
      "\n",
      "Epoch 00029: val_loss did not improve\n",
      "Epoch 30/100\n",
      "122/122 [==============================] - 89s 729ms/step - loss: 0.0021 - acc: 0.9974 - f1: 0.9486 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9080\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.00317 to 0.00316, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 31/100\n",
      "122/122 [==============================] - 90s 735ms/step - loss: 0.0021 - acc: 0.9974 - f1: 0.9492 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9080\n",
      "\n",
      "Epoch 00031: val_loss improved from 0.00316 to 0.00316, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 32/100\n",
      "122/122 [==============================] - 90s 734ms/step - loss: 0.0021 - acc: 0.9974 - f1: 0.9493 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9079\n",
      "\n",
      "Epoch 00032: val_loss improved from 0.00316 to 0.00315, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 33/100\n",
      "122/122 [==============================] - 89s 732ms/step - loss: 0.0021 - acc: 0.9975 - f1: 0.9500 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9082\n",
      "\n",
      "Epoch 00033: val_loss improved from 0.00315 to 0.00315, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 34/100\n",
      "122/122 [==============================] - 90s 741ms/step - loss: 0.0021 - acc: 0.9975 - f1: 0.9504 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9077\n",
      "\n",
      "Epoch 00034: val_loss did not improve\n",
      "Epoch 35/100\n",
      "122/122 [==============================] - 89s 731ms/step - loss: 0.0021 - acc: 0.9975 - f1: 0.9507 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9099\n",
      "\n",
      "Epoch 00035: val_loss improved from 0.00315 to 0.00313, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 36/100\n",
      "122/122 [==============================] - 90s 740ms/step - loss: 0.0020 - acc: 0.9975 - f1: 0.9512 - val_loss: 0.0032 - val_acc: 0.9949 - val_f1: 0.9083\n",
      "\n",
      "Epoch 00036: val_loss did not improve\n",
      "Epoch 37/100\n",
      "122/122 [==============================] - 90s 736ms/step - loss: 0.0020 - acc: 0.9975 - f1: 0.9511 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9094\n",
      "\n",
      "Epoch 00037: val_loss improved from 0.00313 to 0.00313, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 38/100\n",
      "122/122 [==============================] - 90s 734ms/step - loss: 0.0020 - acc: 0.9975 - f1: 0.9519 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9097\n",
      "\n",
      "Epoch 00038: val_loss improved from 0.00313 to 0.00313, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 39/100\n",
      "122/122 [==============================] - 90s 734ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9522 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9096\n",
      "\n",
      "Epoch 00039: val_loss did not improve\n",
      "Epoch 40/100\n",
      "122/122 [==============================] - 93s 763ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9528 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9098\n",
      "\n",
      "Epoch 00040: val_loss did not improve\n",
      "Epoch 41/100\n",
      "122/122 [==============================] - 88s 724ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9530 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9088\n",
      "\n",
      "Epoch 00041: val_loss improved from 0.00313 to 0.00312, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 42/100\n",
      "122/122 [==============================] - 90s 734ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9525 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9094\n",
      "\n",
      "Epoch 00042: val_loss improved from 0.00312 to 0.00312, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 43/100\n",
      "122/122 [==============================] - 89s 733ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9533 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9085\n",
      "\n",
      "Epoch 00043: val_loss improved from 0.00312 to 0.00312, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 44/100\n",
      "122/122 [==============================] - 96s 789ms/step - loss: 0.0020 - acc: 0.9976 - f1: 0.9538 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9097\n",
      "\n",
      "Epoch 00044: val_loss improved from 0.00312 to 0.00311, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 45/100\n",
      "122/122 [==============================] - 94s 774ms/step - loss: 0.0020 - acc: 0.9977 - f1: 0.9539 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9092\n",
      "\n",
      "Epoch 00045: val_loss did not improve\n",
      "Epoch 46/100\n",
      "122/122 [==============================] - 98s 801ms/step - loss: 0.0019 - acc: 0.9977 - f1: 0.9547 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9083\n",
      "\n",
      "Epoch 00046: val_loss did not improve\n",
      "Epoch 47/100\n",
      "122/122 [==============================] - 94s 767ms/step - loss: 0.0019 - acc: 0.9977 - f1: 0.9550 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9095\n",
      "\n",
      "Epoch 00047: val_loss improved from 0.00311 to 0.00310, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 48/100\n",
      "122/122 [==============================] - 91s 749ms/step - loss: 0.0019 - acc: 0.9977 - f1: 0.9546 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9088\n",
      "\n",
      "Epoch 00048: val_loss did not improve\n",
      "Epoch 49/100\n",
      "122/122 [==============================] - 92s 753ms/step - loss: 0.0019 - acc: 0.9977 - f1: 0.9553 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9099\n",
      "\n",
      "Epoch 00049: val_loss improved from 0.00310 to 0.00309, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 50/100\n",
      "122/122 [==============================] - 94s 773ms/step - loss: 0.0019 - acc: 0.9978 - f1: 0.9561 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9110\n",
      "\n",
      "Epoch 00050: val_loss improved from 0.00309 to 0.00308, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 51/100\n",
      "122/122 [==============================] - 90s 740ms/step - loss: 0.0019 - acc: 0.9978 - f1: 0.9563 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9095\n",
      "\n",
      "Epoch 00051: val_loss did not improve\n",
      "Epoch 52/100\n",
      "122/122 [==============================] - 92s 756ms/step - loss: 0.0019 - acc: 0.9978 - f1: 0.9569 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9100\n",
      "\n",
      "Epoch 00052: val_loss did not improve\n",
      "Epoch 53/100\n",
      "122/122 [==============================] - 95s 780ms/step - loss: 0.0019 - acc: 0.9978 - f1: 0.9568 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9112\n",
      "\n",
      "Epoch 00053: val_loss improved from 0.00308 to 0.00308, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 54/100\n",
      "122/122 [==============================] - 94s 769ms/step - loss: 0.0019 - acc: 0.9979 - f1: 0.9578 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9101\n",
      "\n",
      "Epoch 00054: val_loss did not improve\n",
      "Epoch 55/100\n",
      "122/122 [==============================] - 97s 796ms/step - loss: 0.0018 - acc: 0.9979 - f1: 0.9579 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9100\n",
      "\n",
      "Epoch 00055: val_loss did not improve\n",
      "Epoch 56/100\n",
      "122/122 [==============================] - 97s 799ms/step - loss: 0.0018 - acc: 0.9979 - f1: 0.9586 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9098\n",
      "\n",
      "Epoch 00056: val_loss did not improve\n",
      "Epoch 57/100\n",
      "122/122 [==============================] - 93s 763ms/step - loss: 0.0018 - acc: 0.9979 - f1: 0.9585 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9097\n",
      "\n",
      "Epoch 00057: val_loss did not improve\n",
      "Epoch 58/100\n",
      "122/122 [==============================] - 92s 753ms/step - loss: 0.0018 - acc: 0.9979 - f1: 0.9585 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9110\n",
      "\n",
      "Epoch 00058: val_loss improved from 0.00308 to 0.00306, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 59/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0018 - acc: 0.9980 - f1: 0.9596 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9115\n",
      "\n",
      "Epoch 00059: val_loss improved from 0.00306 to 0.00305, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 60/100\n",
      "122/122 [==============================] - 92s 755ms/step - loss: 0.0018 - acc: 0.9979 - f1: 0.9594 - val_loss: 0.0030 - val_acc: 0.9951 - val_f1: 0.9121\n",
      "\n",
      "Epoch 00060: val_loss improved from 0.00305 to 0.00303, saving model to models/v-model-combined-fold-1_100epoch.h5\n",
      "Epoch 61/100\n",
      "122/122 [==============================] - 94s 768ms/step - loss: 0.0018 - acc: 0.9980 - f1: 0.9598 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9124\n",
      "\n",
      "Epoch 00061: val_loss did not improve\n",
      "Epoch 62/100\n",
      "122/122 [==============================] - 95s 778ms/step - loss: 0.0018 - acc: 0.9980 - f1: 0.9599 - val_loss: 0.0031 - val_acc: 0.9950 - val_f1: 0.9110\n",
      "\n",
      "Epoch 00062: val_loss did not improve\n",
      "Epoch 63/100\n",
      "122/122 [==============================] - 93s 763ms/step - loss: 0.0018 - acc: 0.9980 - f1: 0.9607 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9100\n",
      "\n",
      "Epoch 00063: val_loss did not improve\n",
      "Epoch 64/100\n",
      "122/122 [==============================] - 93s 760ms/step - loss: 0.0018 - acc: 0.9980 - f1: 0.9611 - val_loss: 0.0030 - val_acc: 0.9950 - val_f1: 0.9122\n",
      "\n",
      "Epoch 00064: val_loss did not improve\n",
      "Epoch 65/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0017 - acc: 0.9981 - f1: 0.9615 - val_loss: 0.0031 - val_acc: 0.9949 - val_f1: 0.9103\n",
      "\n",
      "Epoch 00065: val_loss did not improve\n",
      "Epoch 00065: early stopping\n",
      "61/61 [==============================] - 13s 219ms/step\n",
      "61/61 [==============================] - 14s 222ms/step\n",
      "Training for fold 2\n",
      "size of LOADED weights: (118,)\n",
      "[[[-0.24358654 -0.16099751 -0.23479125  0.14088532  0.03075149\n",
      "    0.03578469 -0.3162213   0.27086827 -0.07013895 -0.40897346\n",
      "   -0.34079093 -0.15332791 -0.15636504 -0.17843118 -0.13940684\n",
      "    0.38757318]]\n",
      "\n",
      " [[ 0.07683621  0.12581234 -0.0449182  -0.29350325  0.06272634\n",
      "   -0.00982578 -0.04296639  0.19984654  0.04635777  0.03844686\n",
      "   -0.15665068  0.2846878   0.25253665  0.35006022 -0.04413303\n",
      "   -0.05179099]]\n",
      "\n",
      " [[ 0.03134132 -0.18344997  0.22957772  0.03939808  0.17374802\n",
      "    0.11309635  0.09672907 -0.32576624  0.01907272  0.36521757\n",
      "   -0.5326515   0.01514594  0.46846372 -0.44124842  0.06671178\n",
      "    0.10490794]]]\n",
      "Train on 122 samples, validate on 61 samples\n",
      "Epoch 1/100\n",
      "122/122 [==============================] - 94s 768ms/step - loss: 0.0072 - acc: 0.9909 - f1: 0.8209 - val_loss: 0.0067 - val_acc: 0.9903 - val_f1: 0.7914\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.00672, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 2/100\n",
      "122/122 [==============================] - 92s 756ms/step - loss: 0.0056 - acc: 0.9925 - f1: 0.8541 - val_loss: 0.0062 - val_acc: 0.9905 - val_f1: 0.7955\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.00672 to 0.00620, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 3/100\n",
      "122/122 [==============================] - 91s 749ms/step - loss: 0.0054 - acc: 0.9928 - f1: 0.8618 - val_loss: 0.0056 - val_acc: 0.9919 - val_f1: 0.8231\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.00620 to 0.00556, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 4/100\n",
      "122/122 [==============================] - 92s 751ms/step - loss: 0.0048 - acc: 0.9934 - f1: 0.8737 - val_loss: 0.0062 - val_acc: 0.9898 - val_f1: 0.7756\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "122/122 [==============================] - 88s 722ms/step - loss: 0.0054 - acc: 0.9926 - f1: 0.8589 - val_loss: 0.0057 - val_acc: 0.9923 - val_f1: 0.8465\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "122/122 [==============================] - 93s 766ms/step - loss: 0.0046 - acc: 0.9939 - f1: 0.8831 - val_loss: 0.0069 - val_acc: 0.9883 - val_f1: 0.7294\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "\n",
      "Epoch 00006: val_loss did not improve\n",
      "Epoch 7/100\n",
      "122/122 [==============================] - 92s 752ms/step - loss: 0.0039 - acc: 0.9951 - f1: 0.9051 - val_loss: 0.0045 - val_acc: 0.9932 - val_f1: 0.8605\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.00556 to 0.00452, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 8/100\n",
      "122/122 [==============================] - 94s 768ms/step - loss: 0.0036 - acc: 0.9956 - f1: 0.9150 - val_loss: 0.0043 - val_acc: 0.9938 - val_f1: 0.8756\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.00452 to 0.00427, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 9/100\n",
      "122/122 [==============================] - 91s 745ms/step - loss: 0.0035 - acc: 0.9957 - f1: 0.9179 - val_loss: 0.0042 - val_acc: 0.9939 - val_f1: 0.8784\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.00427 to 0.00416, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 10/100\n",
      "122/122 [==============================] - 91s 744ms/step - loss: 0.0034 - acc: 0.9959 - f1: 0.9213 - val_loss: 0.0041 - val_acc: 0.9940 - val_f1: 0.8812\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.00416 to 0.00406, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 11/100\n",
      "122/122 [==============================] - 91s 748ms/step - loss: 0.0033 - acc: 0.9961 - f1: 0.9247 - val_loss: 0.0040 - val_acc: 0.9940 - val_f1: 0.8802\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.00406 to 0.00405, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 12/100\n",
      "122/122 [==============================] - 94s 770ms/step - loss: 0.0032 - acc: 0.9961 - f1: 0.9251 - val_loss: 0.0040 - val_acc: 0.9942 - val_f1: 0.8842\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.00405 to 0.00398, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 13/100\n",
      "122/122 [==============================] - 93s 759ms/step - loss: 0.0031 - acc: 0.9962 - f1: 0.9267 - val_loss: 0.0040 - val_acc: 0.9941 - val_f1: 0.8800\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.00398 to 0.00397, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 14/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0030 - acc: 0.9964 - f1: 0.9313 - val_loss: 0.0038 - val_acc: 0.9944 - val_f1: 0.8894\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.00397 to 0.00381, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 15/100\n",
      "122/122 [==============================] - 93s 760ms/step - loss: 0.0030 - acc: 0.9965 - f1: 0.9327 - val_loss: 0.0038 - val_acc: 0.9945 - val_f1: 0.8913\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.00381 to 0.00378, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 16/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0029 - acc: 0.9965 - f1: 0.9332 - val_loss: 0.0038 - val_acc: 0.9946 - val_f1: 0.8937\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.00378 to 0.00375, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 17/100\n",
      "122/122 [==============================] - 93s 759ms/step - loss: 0.0029 - acc: 0.9966 - f1: 0.9345 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8943\n",
      "\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1e-05.\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.00375 to 0.00374, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 18/100\n",
      "122/122 [==============================] - 94s 772ms/step - loss: 0.0029 - acc: 0.9966 - f1: 0.9345 - val_loss: 0.0037 - val_acc: 0.9945 - val_f1: 0.8929\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.00374 to 0.00374, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 19/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0029 - acc: 0.9966 - f1: 0.9350 - val_loss: 0.0038 - val_acc: 0.9945 - val_f1: 0.8923\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/100\n",
      "122/122 [==============================] - 91s 745ms/step - loss: 0.0029 - acc: 0.9966 - f1: 0.9347 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8947\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.00374 to 0.00372, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 21/100\n",
      "122/122 [==============================] - 93s 759ms/step - loss: 0.0029 - acc: 0.9967 - f1: 0.9363 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8930\n",
      "\n",
      "Epoch 00021: val_loss did not improve\n",
      "Epoch 22/100\n",
      "122/122 [==============================] - 89s 728ms/step - loss: 0.0029 - acc: 0.9967 - f1: 0.9358 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8954\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.00372 to 0.00371, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 23/100\n",
      "122/122 [==============================] - 91s 748ms/step - loss: 0.0029 - acc: 0.9967 - f1: 0.9366 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8950\n",
      "\n",
      "Epoch 00023: val_loss improved from 0.00371 to 0.00370, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 24/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0028 - acc: 0.9967 - f1: 0.9375 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8949\n",
      "\n",
      "Epoch 00024: val_loss did not improve\n",
      "Epoch 25/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0028 - acc: 0.9968 - f1: 0.9377 - val_loss: 0.0037 - val_acc: 0.9947 - val_f1: 0.8960\n",
      "\n",
      "Epoch 00025: val_loss improved from 0.00370 to 0.00368, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 26/100\n",
      "122/122 [==============================] - 90s 737ms/step - loss: 0.0028 - acc: 0.9968 - f1: 0.9375 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8945\n",
      "\n",
      "Epoch 00026: val_loss did not improve\n",
      "Epoch 27/100\n",
      "122/122 [==============================] - 88s 725ms/step - loss: 0.0028 - acc: 0.9968 - f1: 0.9380 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8947\n",
      "\n",
      "Epoch 00027: val_loss did not improve\n",
      "Epoch 28/100\n",
      "122/122 [==============================] - 89s 727ms/step - loss: 0.0028 - acc: 0.9968 - f1: 0.9389 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8943\n",
      "\n",
      "Epoch 00028: val_loss did not improve\n",
      "Epoch 29/100\n",
      "122/122 [==============================] - 87s 715ms/step - loss: 0.0028 - acc: 0.9968 - f1: 0.9392 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8939\n",
      "\n",
      "Epoch 00029: val_loss did not improve\n",
      "Epoch 30/100\n",
      "122/122 [==============================] - 89s 726ms/step - loss: 0.0028 - acc: 0.9969 - f1: 0.9398 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8941\n",
      "\n",
      "Epoch 00030: val_loss improved from 0.00368 to 0.00367, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 31/100\n",
      "122/122 [==============================] - 89s 731ms/step - loss: 0.0027 - acc: 0.9969 - f1: 0.9401 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8964\n",
      "\n",
      "Epoch 00031: val_loss improved from 0.00367 to 0.00365, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 32/100\n",
      "122/122 [==============================] - 89s 734ms/step - loss: 0.0027 - acc: 0.9969 - f1: 0.9410 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8950\n",
      "\n",
      "Epoch 00032: val_loss did not improve\n",
      "Epoch 33/100\n",
      "122/122 [==============================] - 89s 727ms/step - loss: 0.0027 - acc: 0.9969 - f1: 0.9408 - val_loss: 0.0037 - val_acc: 0.9946 - val_f1: 0.8943\n",
      "\n",
      "Epoch 00033: val_loss did not improve\n",
      "Epoch 34/100\n",
      "122/122 [==============================] - 89s 726ms/step - loss: 0.0027 - acc: 0.9970 - f1: 0.9418 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8979\n",
      "\n",
      "Epoch 00034: val_loss improved from 0.00365 to 0.00361, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 35/100\n",
      "122/122 [==============================] - 89s 730ms/step - loss: 0.0027 - acc: 0.9970 - f1: 0.9424 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8951\n",
      "\n",
      "Epoch 00035: val_loss did not improve\n",
      "Epoch 36/100\n",
      "122/122 [==============================] - 88s 723ms/step - loss: 0.0027 - acc: 0.9970 - f1: 0.9427 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8983\n",
      "\n",
      "Epoch 00036: val_loss improved from 0.00361 to 0.00358, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 37/100\n",
      "122/122 [==============================] - 92s 755ms/step - loss: 0.0026 - acc: 0.9971 - f1: 0.9433 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8971\n",
      "\n",
      "Epoch 00037: val_loss did not improve\n",
      "Epoch 38/100\n",
      "122/122 [==============================] - 95s 778ms/step - loss: 0.0026 - acc: 0.9971 - f1: 0.9437 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8954\n",
      "\n",
      "Epoch 00038: val_loss did not improve\n",
      "Epoch 39/100\n",
      "122/122 [==============================] - 92s 750ms/step - loss: 0.0026 - acc: 0.9971 - f1: 0.9439 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8969\n",
      "\n",
      "Epoch 00039: val_loss did not improve\n",
      "Epoch 40/100\n",
      "122/122 [==============================] - 92s 758ms/step - loss: 0.0026 - acc: 0.9971 - f1: 0.9453 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8955\n",
      "\n",
      "Epoch 00040: val_loss did not improve\n",
      "Epoch 41/100\n",
      "122/122 [==============================] - 92s 756ms/step - loss: 0.0026 - acc: 0.9972 - f1: 0.9450 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8979\n",
      "\n",
      "Epoch 00041: val_loss improved from 0.00358 to 0.00357, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 42/100\n",
      "122/122 [==============================] - 94s 769ms/step - loss: 0.0026 - acc: 0.9972 - f1: 0.9458 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8991\n",
      "\n",
      "Epoch 00042: val_loss improved from 0.00357 to 0.00356, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 43/100\n",
      "122/122 [==============================] - 95s 775ms/step - loss: 0.0026 - acc: 0.9972 - f1: 0.9463 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8981\n",
      "\n",
      "Epoch 00043: val_loss did not improve\n",
      "Epoch 44/100\n",
      "122/122 [==============================] - 90s 734ms/step - loss: 0.0025 - acc: 0.9973 - f1: 0.9475 - val_loss: 0.0036 - val_acc: 0.9947 - val_f1: 0.8968\n",
      "\n",
      "Epoch 00044: val_loss did not improve\n",
      "Epoch 45/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0025 - acc: 0.9973 - f1: 0.9478 - val_loss: 0.0036 - val_acc: 0.9948 - val_f1: 0.8968\n",
      "\n",
      "Epoch 00045: val_loss did not improve\n",
      "Epoch 46/100\n",
      "122/122 [==============================] - 98s 800ms/step - loss: 0.0025 - acc: 0.9973 - f1: 0.9481 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8994\n",
      "\n",
      "Epoch 00046: val_loss improved from 0.00356 to 0.00353, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 47/100\n",
      "122/122 [==============================] - 95s 776ms/step - loss: 0.0025 - acc: 0.9974 - f1: 0.9489 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9008\n",
      "\n",
      "Epoch 00047: val_loss improved from 0.00353 to 0.00352, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 48/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0025 - acc: 0.9974 - f1: 0.9501 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8989\n",
      "\n",
      "Epoch 00048: val_loss did not improve\n",
      "Epoch 49/100\n",
      "122/122 [==============================] - 97s 798ms/step - loss: 0.0024 - acc: 0.9974 - f1: 0.9503 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8989\n",
      "\n",
      "Epoch 00049: val_loss did not improve\n",
      "Epoch 50/100\n",
      "122/122 [==============================] - 96s 783ms/step - loss: 0.0024 - acc: 0.9974 - f1: 0.9501 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8990\n",
      "\n",
      "Epoch 00050: val_loss improved from 0.00352 to 0.00352, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 51/100\n",
      "122/122 [==============================] - 94s 774ms/step - loss: 0.0024 - acc: 0.9975 - f1: 0.9515 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.9000\n",
      "\n",
      "Epoch 00051: val_loss did not improve\n",
      "Epoch 52/100\n",
      "122/122 [==============================] - 93s 759ms/step - loss: 0.0024 - acc: 0.9975 - f1: 0.9517 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8994\n",
      "\n",
      "Epoch 00052: val_loss improved from 0.00352 to 0.00351, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 53/100\n",
      "122/122 [==============================] - 96s 787ms/step - loss: 0.0024 - acc: 0.9975 - f1: 0.9521 - val_loss: 0.0035 - val_acc: 0.9948 - val_f1: 0.8992\n",
      "\n",
      "Epoch 00053: val_loss did not improve\n",
      "Epoch 54/100\n",
      "122/122 [==============================] - 94s 775ms/step - loss: 0.0024 - acc: 0.9975 - f1: 0.9522 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9012\n",
      "\n",
      "Epoch 00054: val_loss improved from 0.00351 to 0.00348, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 55/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0024 - acc: 0.9976 - f1: 0.9530 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9003\n",
      "\n",
      "Epoch 00055: val_loss did not improve\n",
      "Epoch 56/100\n",
      "122/122 [==============================] - 93s 759ms/step - loss: 0.0023 - acc: 0.9976 - f1: 0.9538 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.8997\n",
      "\n",
      "Epoch 00056: val_loss did not improve\n",
      "Epoch 57/100\n",
      "122/122 [==============================] - 94s 774ms/step - loss: 0.0023 - acc: 0.9977 - f1: 0.9547 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9020\n",
      "\n",
      "Epoch 00057: val_loss improved from 0.00348 to 0.00348, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 58/100\n",
      "122/122 [==============================] - 94s 768ms/step - loss: 0.0023 - acc: 0.9977 - f1: 0.9549 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9020\n",
      "\n",
      "Epoch 00058: val_loss improved from 0.00348 to 0.00346, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 59/100\n",
      "122/122 [==============================] - 95s 782ms/step - loss: 0.0023 - acc: 0.9977 - f1: 0.9557 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9020\n",
      "\n",
      "Epoch 00059: val_loss did not improve\n",
      "Epoch 60/100\n",
      "122/122 [==============================] - 93s 762ms/step - loss: 0.0023 - acc: 0.9977 - f1: 0.9557 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9008\n",
      "\n",
      "Epoch 00060: val_loss did not improve\n",
      "Epoch 61/100\n",
      "122/122 [==============================] - 91s 748ms/step - loss: 0.0023 - acc: 0.9978 - f1: 0.9568 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9006\n",
      "\n",
      "Epoch 00061: val_loss did not improve\n",
      "Epoch 62/100\n",
      "122/122 [==============================] - 94s 770ms/step - loss: 0.0023 - acc: 0.9978 - f1: 0.9565 - val_loss: 0.0035 - val_acc: 0.9949 - val_f1: 0.9013\n",
      "\n",
      "Epoch 00062: val_loss did not improve\n",
      "Epoch 63/100\n",
      "122/122 [==============================] - 97s 793ms/step - loss: 0.0022 - acc: 0.9978 - f1: 0.9575 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9049\n",
      "\n",
      "Epoch 00063: val_loss improved from 0.00346 to 0.00341, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 64/100\n",
      "122/122 [==============================] - 96s 789ms/step - loss: 0.0022 - acc: 0.9978 - f1: 0.9578 - val_loss: 0.0034 - val_acc: 0.9951 - val_f1: 0.9061\n",
      "\n",
      "Epoch 00064: val_loss improved from 0.00341 to 0.00339, saving model to models/v-model-combined-fold-2_100epoch.h5\n",
      "Epoch 65/100\n",
      "122/122 [==============================] - 93s 764ms/step - loss: 0.0022 - acc: 0.9978 - f1: 0.9583 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9006\n",
      "\n",
      "Epoch 00065: val_loss did not improve\n",
      "Epoch 66/100\n",
      "122/122 [==============================] - 92s 753ms/step - loss: 0.0022 - acc: 0.9979 - f1: 0.9591 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9036\n",
      "\n",
      "Epoch 00066: val_loss did not improve\n",
      "Epoch 67/100\n",
      "122/122 [==============================] - 92s 752ms/step - loss: 0.0022 - acc: 0.9979 - f1: 0.9593 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9038\n",
      "\n",
      "Epoch 00067: val_loss did not improve\n",
      "Epoch 68/100\n",
      "122/122 [==============================] - 92s 754ms/step - loss: 0.0022 - acc: 0.9979 - f1: 0.9595 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9054\n",
      "\n",
      "Epoch 00068: val_loss did not improve\n",
      "Epoch 69/100\n",
      "122/122 [==============================] - 95s 776ms/step - loss: 0.0021 - acc: 0.9980 - f1: 0.9610 - val_loss: 0.0034 - val_acc: 0.9950 - val_f1: 0.9039\n",
      "\n",
      "Epoch 00069: val_loss did not improve\n",
      "Epoch 00069: early stopping\n",
      "61/61 [==============================] - 13s 217ms/step\n",
      "61/61 [==============================] - 14s 234ms/step\n",
      "elapsed time 17135.020541906357\n"
     ]
    }
   ],
   "source": [
    "t = time.time()\n",
    "for j, (train,val) in enumerate(kfolds.split(x, y_reshaped)):\n",
    "    print('Training for fold {j}'.format(j=j))\n",
    "    input_img = Input((nx, ny, 1), name='img')\n",
    "    \n",
    "    model = create_unet(input_img, n_filters=16, dropout=0.025, batch_norm=True)\n",
    "    model.compile(optimizer=Adam(), loss=dice_coef_loss, metrics=[\"accuracy\", f1])\n",
    "    \n",
    "    ## OLD MODEL Weights file HERE!!   \n",
    "    if LoadPrevious == 1:\n",
    "        #model.load_weights('./models/'+source_string[0]+'-model-geophone-fold-{i}_50epoch.h5'.format(i=j))\n",
    "        # # MICHAEL'S WEIGHTS: VERTICAL ONLY\n",
    "        model.load_weights('./models/model-'+receiver_string+'-dice-fold-{i}-keras.h5'.format(i=j))\n",
    "        # #\n",
    "        print(\"size of LOADED weights:\",np.shape(model.get_weights()))\n",
    "        print(model.get_weights()[0][0])\n",
    "        \n",
    "    ## NEW MODEL Weights file HERE!!    \n",
    "    callbacks = [\n",
    "        EarlyStopping(monitor='val_loss', patience=5, verbose=1),\n",
    "        ReduceLROnPlateau(factor=0.1, patience=2, min_lr=0.00001, verbose=1),\n",
    "        ModelCheckpoint(NameCheckpoint+'-{i}_100epoch.h5'.format(i=j), verbose=1, save_best_only=True, save_weights_only=True)\n",
    "    ]\n",
    "    \n",
    "    results = model.fit(x[train], y[train], batch_size=1, epochs=100, callbacks=callbacks,\n",
    "                       validation_data=(x[val], y[val]))\n",
    "    \n",
    "    eval_stats.append(model.evaluate(x[val], y[val], verbose=1))    \n",
    "    # Predict on train, val and test\n",
    "    preds_val = model.predict(x[val], verbose=1)\n",
    "    # Threshold predictions\n",
    "    preds_val_t = (preds_val > 0.5).astype(np.uint8)\n",
    "    # confusion matrix calculations\n",
    "    c_matrix = confusion_matrix(y[val].ravel(), preds_val_t.ravel())\n",
    "    f1_metric = f1_score(y[val].ravel(), preds_val_t.ravel())\n",
    "    recall = recall_score(y[val].ravel(), preds_val_t.ravel())\n",
    "    precision = precision_score(y[val].ravel(), preds_val_t.ravel())\n",
    "    c_matrices.append(c_matrix)\n",
    "    f1_stats.append(f1_metric)\n",
    "    recall_stats.append(recall)\n",
    "    precision_stats.append(precision)\n",
    "    \n",
    "elapsed = time.time() - t\n",
    "print('elapsed time',elapsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0032651767050694736, 0.9949603432514629, 0.8967932628803565], [0.003087607678025961, 0.9949308203869178, 0.9114002683123604], [0.0034024900237678503, 0.9950241329240017, 0.9078158355150067]]\n",
      "[array([[3885713,    6786],\n",
      "       [  13361,   91836]]), array([[3874677,    8572],\n",
      "       [  11693,  102754]]), array([[3877592,    8506],\n",
      "       [  11386,  100212]])]\n",
      "[0.9011524931434263, 0.9102417029494227, 0.9097115052923983]\n",
      "[0.8729906746390106, 0.8978304367960716, 0.8979730819548738]\n",
      "[0.9311918233254244, 0.9230009162280151, 0.9217608859618462]\n"
     ]
    }
   ],
   "source": [
    "print(eval_stats)\n",
    "print(c_matrices)\n",
    "print(f1_stats)\n",
    "print(recall_stats)\n",
    "print(precision_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c_matrices_stacked = np.rollaxis(np.dstack(c_matrices), axis=-1)\n",
    "eval_stats = np.asarray(eval_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(69,)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(results.history[\"loss\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAI4CAYAAABndZP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcHVWd9/HP7/aSzgYhCAEShACBsIS1QRABAZXFhdFR\nEUG2EQbcZ9wYHRR95NHheUZ9mEEYVBQVFAZQcWBkVKKIiCaBQEAWA4aQEENYErL1du95/qi66dtN\nSNKd7r7p6s/79bqvW7fqVNW5JxfyzTmnqiKlhCRJUpGU6l0BSZKkgWbAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkbRFi4j/joiz6l0PScNLeB8cSesTEQuA96eUflnvukhSX9mDI6luIqKx\n3nXYXEX4DlIRGXAk9VlEvCUi5kbE8oi4JyL2r9l2UUQ8ERErI+JPEfH2mm1nR8TvIuJrEfE8cEm+\n7u6I+L8R8WJE/CUiTqrZ59cR8f6a/TdUdmpE3JWf+5cRcUVE/GAD3+OU/Hu8lNf5xHz9goh4Q025\nS6rHiYhdIyJFxN9FxELgznwY7UO9jv1ARLwjX54eEb+IiBci4rGIeHf/W1/SpjDgSOqTiDgIuAb4\ne2Bb4D+AWyNiVF7kCeAoYGvgC8APImLHmkO8BngSmARcWrPuMeBVwGXAtyMiXqEKGyp7PfDHvF6X\nAO/bwPc4DPge8ElgAnA0sGBj37/GMcDewAnAD4HTao69D7ALcFtEjAV+kddte+A9wDfyMpIGiQFH\nUl+dD/xHSukPKaVySulaoB04HCCl9J8ppWdSSpWU0g3An4HDavZ/JqX0bymlrpTS2nzdUymlb6aU\nysC1wI5kAWh91ls2Il4NHAp8LqXUkVK6G7h1A9/j74BrUkq/yOu6OKX0aB/a4ZKU0ur8O/wYODAi\ndsm3nQ7cklJqB94CLEgpfSf/zvcDNwPv6sO5JPWRAUdSX+0CfDwfnloeEcuBnYGdACLizJrhq+XA\nfmS9LVVPr+eYf60upJTW5IvjXuH8r1R2J+CFmnWvdK6qncl6m/pr3bFTSiuB28h6ZyDrzbkuX94F\neE2v9jod2GEzzi1pI5wcJ6mvngYuTSld2ntD3oPxTeB44PcppXJEzAVqh5sG69LNJcDEiBhTE3J2\n3kD5p4HdX2HbamBMzef1hZHe3+OHwOcj4i6gBZhZc57fpJTeuKHKSxpY9uBI2pCmiGipeTWSBZgL\nIuI1kRkbEW+OiPHAWLK/+JcBRMQ5ZD04gy6l9BQwm2zicnNEHAG8dQO7fBs4JyKOj4hSREyOiOn5\ntrnAeyKiKSJagXduQhVuJ+ut+SJwQ0qpkq//L2DPiHhffrymiDg0Ivbuz/eUtGkMOJI25HZgbc3r\nkpTSbOA84N+BF4H5wNkAKaU/Af8K/B5YCswAfjeE9T0dOAJ4HvgScAPZ/KCXSSn9ETgH+BqwAvgN\nWUABuJisd+dFsonS12/sxPl8m1uAN9SWz4ev3kQ2fPUM2RDbvwCj1nMYSQPEG/1JKqyIuAF4NKX0\n+XrXRdLQsgdHUmHkQz+750NOJwKnAD+pd70kDb26BJyIODG/2dX8iLhoPdsjIi7Ptz8YEQfXbFsQ\nEfPyqzRmD23NJW3hdgB+DawCLgcuzC/LljTCDPkQVUQ0AI8DbwQWAbOA0/Kx+2qZk4EPAyeT3dTr\n/6WUXpNvWwC0ppSeG9KKS5KkYaMePTiHAfNTSk+mlDqAH5F1I9c6BfheytwLTOh1J1RJkqRXVI/7\n4Eym5823FpH10myszGSy+1wk4JcRUSa7m+rV6ztJRJxPdsdVxo4de8j06dPXV0ySJA0jc+bMeS6l\ntN3Gyg3HG/29LqW0OCK2B34REY+mlO7qXSgPPlcDtLa2ptmzna4jSdJwFxFPbUq5egxRLabn3UWn\n5Os2qUxKqfr+LNnzXw5DkiSpRj0CzixgWkRMjYhmsptf9X4g3q3AmfnVVIcDK1JKS/I7po4HyJ/Q\n+ybgoaGsvCRJ2vIN+RBVSqkrIj4E3AE0kD3N9+GIuCDffhXZ3VNPJrtD6hqyu41C9nThH0dEte7X\np5R+PsRfQZIkbeFGxJ2MnYMjSRoInZ2dLFq0iLa2tnpXpfBaWlqYMmUKTU1NPdZHxJyUUuvG9h+O\nk4wlSaqLRYsWMX78eHbddVfy0QQNgpQSzz//PIsWLWLq1Kn9OoaPapAkaRO1tbWx7bbbGm4GWUSw\n7bbbblZPmQFHkqQ+MNwMjc1tZwOOJEkqHAOOJEnDyLhx4+pdhWHBgCNJkgrHgCNJ0jCUUuKTn/wk\n++23HzNmzOCGG24AYMmSJRx99NEceOCB7Lfffvz2t7+lXC5z9tlnryv7ta99rc61H3xeJi5JUj98\n4WcP86dnXhrQY+6z01Z8/q37blLZW265hblz5/LAAw/w3HPPceihh3L00Udz/fXXc8IJJ/DZz36W\ncrnMmjVrmDt3LosXL+ahh7Kb/y9fvnxA670lsgdHkqRh6O677+a0006joaGBSZMmccwxxzBr1iwO\nPfRQvvOd73DJJZcwb948xo8fz2677caTTz7Jhz/8YX7+85+z1VZb1bv6g84eHEmS+mFTe1qG2tFH\nH81dd93Fbbfdxtlnn80//uM/cuaZZ/LAAw9wxx13cNVVV3HjjTdyzTXX1Luqg8oeHEmShqGjjjqK\nG264gXK5zLJly7jrrrs47LDDeOqpp5g0aRLnnXce73//+7nvvvt47rnnqFQq/O3f/i1f+tKXuO++\n++pd/UFnD44kScPQ29/+dn7/+99zwAEHEBFcdtll7LDDDlx77bX8n//zf2hqamLcuHF873vfY/Hi\nxZxzzjlUKhUAvvzlL9e59oPPh21KkrSJHnnkEfbee+96V2PEWF97b+rDNh2ikiRJhWPAkSRJhWPA\nkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSSpwMaNG/eK2xYsWMB+++03hLUZOgYcSZJU\nOD6qQZKk/vjvi+Cv8wb2mDvMgJO+ssEiF110ETvvvDMf/OAHAbjkkktobGxk5syZvPjii3R2dvKl\nL32JU045pU+nbmtr48ILL2T27Nk0Njby1a9+lWOPPZaHH36Yc845h46ODiqVCjfffDM77bQT7373\nu1m0aBHlcpmLL76YU089td9fezAYcCRJGkZOPfVUPvaxj60LODfeeCN33HEHH/nIR9hqq6147rnn\nOPzww3nb295GRGzyca+44goignnz5vHoo4/ypje9iccff5yrrrqKj370o5x++ul0dHRQLpe5/fbb\n2WmnnbjtttsAWLFixaB8181hwJEkqT820tMyWA466CCeffZZnnnmGZYtW8Y222zDDjvswD/8wz9w\n1113USqVWLx4MUuXLmWHHXbY5OPefffdfPjDHwZg+vTp7LLLLjz++OMcccQRXHrppSxatIh3vOMd\nTJs2jRkzZvDxj3+cT3/607zlLW/hqKOOGqyv22/OwZEkaZh517vexU033cQNN9zAqaeeynXXXcey\nZcuYM2cOc+fOZdKkSbS1tQ3Iud773vdy6623Mnr0aE4++WTuvPNO9txzT+677z5mzJjBP//zP/PF\nL35xQM41kOzBkSRpmDn11FM577zzeO655/jNb37DjTfeyPbbb09TUxMzZ87kqaee6vMxjzrqKK67\n7jqOO+44Hn/8cRYuXMhee+3Fk08+yW677cZHPvIRFi5cyIMPPsj06dOZOHEiZ5xxBhMmTOBb3/rW\nIHzLzWPAkSRpmNl3331ZuXIlkydPZscdd+T000/nrW99KzNmzKC1tZXp06f3+Zgf+MAHuPDCC5kx\nYwaNjY1897vfZdSoUdx44418//vfp6mpiR122IHPfOYzzJo1i09+8pOUSiWampq48sorB+Fbbp5I\nKdW7DoOutbU1zZ49u97VkCQNc4888gh77713vasxYqyvvSNiTkqpdWP7OgdHkiQVjkNUkiQV3Lx5\n83jf+97XY92oUaP4wx/+UKcaDT4DjiRJBTdjxgzmzp1b72oMKYeoJElS4RhwJElS4RhwJElS4Rhw\nJEkaJhYsWMB+++232cf59a9/zT333DMANdr4ed7ylrdsdpn+MOBIkjQYLrsMZs7suW7mzGx9nQ1V\nwKknA44kSYPh0EPh3e/uDjkzZ2afDz10sw7b1dXF6aefzt5778073/lO1qxZA8CcOXM45phjOOSQ\nQzjhhBNYsmQJAJdffjn77LMP+++/P+95z3tYsGABV111FV/72tc48MAD+e1vf9vj+JdccglnnXUW\nRx11FLvssgu33HILn/rUp5gxYwYnnnginZ2dAPzqV7/ioIMOYsaMGZx77rm0t7cD8POf/5zp06dz\n8MEHc8stt6w77urVqzn33HM57LDDOOigg/jpT3+6We2wMV4mLklSf3zsY7CxS6932glOOAF23BGW\nLIG994YvfCF7rc+BB8LXv77BQz722GN8+9vf5sgjj+Tcc8/lG9/4Bh/96Ef58Ic/zE9/+lO22247\nbrjhBj772c9yzTXX8JWvfIW//OUvjBo1iuXLlzNhwgQuuOACxo0bxyc+8Yn1nuOJJ55g5syZ/OlP\nf+KII47g5ptv5rLLLuPtb387t912GyeeeCJnn302v/rVr9hzzz0588wzufLKK7ngggs477zzuPPO\nO9ljjz049dRT1x3z0ksv5bjjjuOaa65h+fLlHHbYYbzhDW/YcPttBntwJEkaLNtsk4WbhQuz9222\n2exD7rzzzhx55JEAnHHGGdx999089thjPPTQQ7zxjW/kwAMP5Etf+hKLFi0CYP/99+f000/nBz/4\nAY2Nm9avcdJJJ9HU1MSMGTMol8uceOKJQHY/nQULFvDYY48xdepU9txzTwDOOuss7rrrLh599FGm\nTp3KtGnTiAjOOOOMdcf8n//5H77yla9w4IEH8vrXv562tjYWLly42e3xSuzBkSSpPzbS0wJ0D0td\nfDFceSV8/vNw7LGbddqIeNnnlBL77rsvv//9719W/rbbbuOuu+7iZz/7GZdeeinz5s3b6DlGjRoF\nsO5hmtVzlkolurq6+lXvlBI333wze+21V4/1S5cu7dfxNsYeHEmSBkM13Nx4I3zxi9l77Zycflq4\ncOG6IHP99dfzute9jr322otly5atW9/Z2cnDDz9MpVLh6aef5thjj+Vf/uVfWLFiBatWrWL8+PGs\nXLmy33XYa6+9WLBgAfPnzwfg+9//PscccwzTp09nwYIFPPHEEwD88Ic/XLfPCSecwL/9279Rfcj3\n/fff3+/zbwoDjiRJg2HWrCzUVHtsjj02+zxr1mYddq+99uKKK65g77335sUXX+TCCy+kubmZm266\niU9/+tMccMABHHjggdxzzz2Uy2XOOOMMZsyYwUEHHcRHPvIRJkyYwFvf+lZ+/OMfr3eS8aZoaWnh\nO9/5Du9617uYMWMGpVKJCy64gJaWFq6++mre/OY3c/DBB7P99tuv2+fiiy+ms7OT/fffn3333ZeL\nL754s9phY6KapIqstbU1zZ49u97VkCQNc4888gh77713vasxYqyvvSNiTkqpdWP72oMjSZIKx4Aj\nSZIKx4AjSVIfjISpHVuCzW1nA44kSZuopaWF559/3pAzyFJKPP/887S0tPT7GN4HR5KkTTRlyhQW\nLVrEsmXL6l2VwmtpaWHKlCn93t+AI0nSJmpqamLq1Kn1roY2gUNUkiSpcAw4kiSpcAw4kiSpcAw4\nkiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSp\ncAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4\nkiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSp\ncAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4\nkiSpcOoScCLixIh4LCLmR8RF69keEXF5vv3BiDi41/aGiLg/Iv5r6GotSZKGiyEPOBHRAFwBnATs\nA5wWEfv0KnYSMC1/nQ9c2Wv7R4FHBrmqkiRpmKpHD85hwPyU0pMppQ7gR8ApvcqcAnwvZe4FJkTE\njgARMQV4M/Ctoay0JEkaPuoRcCYDT9d8XpSv29QyXwc+BVQ2dJKIOD8iZkfE7GXLlm1ejSVJ0rAy\nrCYZR8RbgGdTSnM2VjaldHVKqTWl1LrddtsNQe0kSdKWoh4BZzGwc83nKfm6TSlzJPC2iFhANrR1\nXET8YPCqKkmShqN6BJxZwLSImBoRzcB7gFt7lbkVODO/mupwYEVKaUlK6Z9SSlNSSrvm+92ZUjpj\nSGsvSZK2eI1DfcKUUldEfAi4A2gArkkpPRwRF+TbrwJuB04G5gNrgHOGup6SJGn4ipRSvesw6Fpb\nW9Ps2bPrXQ1JkrSZImJOSql1Y+WG1SRjSZKkTWHAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPA\nkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPA\nkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPA\nkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPA\nkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJ\nhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhWPAkSRJhVOX\ngBMRJ0bEYxExPyIuWs/2iIjL8+0PRsTB+fqWiPhjRDwQEQ9HxBeGvvaSJGlLN+QBJyIagCuAk4B9\ngNMiYp9exU4CpuWv84Er8/XtwHEppQOAA4ETI+LwIam4JEkaNurRg3MYMD+l9GRKqQP4EXBKrzKn\nAN9LmXuBCRGxY/55VV6mKX+lIau5JEkaFuoRcCYDT9d8XpSv26QyEdEQEXOBZ4FfpJT+sL6TRMT5\nETE7ImYvW7ZswCovSZK2fMNuknFKqZxSOhCYAhwWEfu9QrmrU0qtKaXW7bbbbmgrKUmS6qoeAWcx\nsHPN5yn5uj6VSSktB2YCJw5CHSVJ0jBWj4AzC5gWEVMjohl4D3BrrzK3AmfmV1MdDqxIKS2JiO0i\nYgJARIwG3gg8OpSVlyRJW77GoT5hSqkrIj4E3AE0ANeklB6OiAvy7VcBtwMnA/OBNcA5+e47Atfm\nV2KVgBtTSv811N9BkiRt2SKl4l+E1NrammbPnl3vakiSpM0UEXNSSq0bKzfsJhlLkiRtjAFHkiQV\njgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFH\nkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVjgFHkiQVzoAEnIiYMBDHkSRJGgh9CjgR\ncWFEfKrm84ERsQh4PiLmRMSUAa+hJElSH/W1B+fDwEs1ny8HngFOz4/1lQGqlyRJUr819rH8q4HH\nACJiO+BI4PiU0q8jogP49wGunyRJUp/1tQenHWjOl48F1gC/zT+/ADgXR5Ik1V1fe3D+CHwwn3fz\nEeDnKaVyvm03suEqSZKkuuprD87HgX2BecDOwGdrtp0K/G6A6iVJktRvferBSSn9Cdg9IrYFXkgp\npZrNnwD+OpCVkyRJ6o++DlEBkFJ6vrocEdsAuwCPpJTaB6pikiRJ/dXX++B8ISK+UvP5OGAhMAd4\nIiL2HeD6SZIk9Vlf5+CcDjxa8/lfgbvJLhd/HPjyANVLkiSp3/oacHYCngSIiJ2BA4DPp5TuJQs7\nhw9s9SRJkvqurwFnJbB1vnwc8GJK6Y/55zZgzEBVTJIkqb/6Osn4N8BFEVEhu2rqpzXb9gSeHqiK\nSZIk9Vdfe3D+gexuxj8CltPzPjhnAncNUL0kSZL6ra/3wVlMNjS1PieQDVNJkiTVVb/ugxMRzcAM\nYCLZM6jmpZRe2vBekiRJQ6OvQ1RExKeApWTPpbojf18aEZ8c4LpJkiT1S596cCLiY2T3urkKuIEs\n6Ewiew7VlyOiPaV0+YDXUpIkqQ/6OkT1QeArKaXaycWPAXdFxHKyJ4wbcCRJUl31dYhqZ2DmK2z7\nNTBls2ojSZI0APoacBYCb3qFbW/Mt0uSJNVVX4eoLgcuj4iJwE1kc3C2B94FnA18dEBrJ0mS1A99\nvQ/Ov0dEO/B54FwgAQE8A1yQUvrWwFdRkiSpb/p8H5yU0jcj4ltk8212BJYAi1JKaaArJ0mS1B/9\nutFfHmaexmdPSZKkLdBGA05EfKAPx0sppSs3oz6SJEmbbVN6cP69D8dLgAFHkiTV1UYDTkqpz49z\nkCRJqqdBCy8RUYqIOyNi2mCdQ5IkaX0Gs3cmgNcD4wfxHJIkSS/j8JMkSSocA44kSSocA44kSSoc\nA44kSSqcwQ44Pr5BkiQNucEOODHIx5ckSXqZfj2LalOklMo4BCZJkuqgTwEnIu7cwOYK8BIwF/hO\nSskHcUqSpLroaw/L88AewOuA0cCq/P11wJ7AWOAjwMMRcegA1lOSJGmT9TXg/BewDNg1pXRESult\nKaUjgKnAc8B/ArsBfwK+PKA1lSRJ2kR9DTifA76YUnqmdmVKaTHwReCzKaWXgK8CrxmYKkqSJPVN\nXwPOjsCoV9jWAkzKl5/FK6gkSVKd9DXg/Ab4SkQcXLsyIlrJhqR+na+aBjy12bWTJEnqh74GnPPJ\nrpSaFRGLI2JuRCwG/gAsB/6+5riXDVw1JUmSNl2fLhPPL/0+MCLeDLQCOwB/BWallG6vKfcfA1pL\nSZKkPujXjf5SSrcBtw1wXSRJkgZEnwNORDQCf0t275uJwAvAb4FbUkpdA1s9SZKkvuvrnYy3B/4H\n2B9YACwFjgA+CDwQEW9KKS0b6EpKkiT1RV8nGX8V2BY4PKW0W36zv93I7nmzbb5dkiSprvoacE4G\nPp1S+mPtypTSLOCfgDcPVMUkSZL6q68BZxSw8hW2rQSaN686kiRJm6+vAede4NMRMbZ2Zf750/l2\nSZKkuurrVVQfB2YCT0fE/5BNMt4eOIHs0QyvH9DaSZIk9UOfenBSSnOBPYGrge2AN5IFnKuAaSml\nBwa8hpIkSX3U5/vg5JeBXzQIdZEkSRoQGw04ETELSJt6wJTSYZtVI0mSpM20KT04D9OHgCNJklRv\nGw04KaWzh6AekiRJA6avl4lLkiRt8Qw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSp\ncAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcAw4kiSpcOoScCLixIh4\nLCLmR8RF69keEXF5vv3BiDg4X79zRMyMiD9FxMMR8dGhr70kSdrSDXnAiYgG4ArgJGAf4LSI2KdX\nsZOAafnrfODKfH0X8PGU0j7A4cAH17OvJEka4erRg3MYMD+l9GRKqQP4EXBKrzKnAN9LmXuBCRGx\nY0ppSUrpPoCU0krgEWDyUFZekiRt+eoRcCYDT9d8XsTLQ8pGy0TErsBBwB/Wd5KIOD8iZkfE7GXL\nlm1mlSVJ0nAyLCcZR8Q44GbgYymll9ZXJqV0dUqpNaXUut122w1tBSVJUl3VI+AsBnau+TwlX7dJ\nZSKiiSzcXJdSumUQ6ylJkoapegScWcC0iJgaEc3Ae4Bbe5W5FTgzv5rqcGBFSmlJRATwbeCRlNJX\nh7bakiRpuGgc6hOmlLoi4kPAHUADcE1K6eGIuCDffhVwO3AyMB9YA5yT734k8D5gXkTMzdd9JqV0\n+1B+B0mStGWLlFK96zDoWltb0+zZs+tdDUmStJkiYk5KqXVj5YblJGNJkqQNMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBI\nkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBI\nkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBI\nkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBI\nkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBI\nkqTCMeBFvV0ZAAAf0UlEQVRIkqTCMeBIkqTCqUvAiYgTI+KxiJgfERetZ3tExOX59gcj4uCabddE\nxLMR8dDQ1lqSJA0XQx5wIqIBuAI4CdgHOC0i9ulV7CRgWv46H7iyZtt3gRMHv6aSJGm4qkcPzmHA\n/JTSkymlDuBHwCm9ypwCfC9l7gUmRMSOACmlu4AXhrTGkiRpWKlHwJkMPF3zeVG+rq9lNigizo+I\n2RExe9myZf2qqCRJGp4KO8k4pXR1Sqk1pdS63Xbb1bs6kiRpCNUj4CwGdq75PCVf19cykiRJ61WP\ngDMLmBYRUyOiGXgPcGuvMrcCZ+ZXUx0OrEgpLRnqikqSpOFpyANOSqkL+BBwB/AIcGNK6eGIuCAi\nLsiL3Q48CcwHvgl8oLp/RPwQ+D2wV0Qsioi/G9IvIEmStniRUqp3HQZda2trmj17dr2rIUmSNlNE\nzEkptW6sXGEnGUuSpJHLgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrH\ngCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJkgrHgCNJ\nkgrHgCNJkgrHgCNJkgrHgCNJkgrHgNNPlUrirseX0dZZrndVJElSLwacfpr91Iucec0f+a8Hl9S7\nKpIkqRcDTj8duus2TNt+HNfes4CUUr2rI0mSahhw+ikiOPO1uzJv8QruW7i8fhV5ehb8+Zf1O78k\nSVsgA85meMdBkxnf0si19yyoTwXaV8ENp8Mt74eKc4EkSaoy4PRXVwdjf/15zjxgPLfPW8KzL7UN\nfR1+9/9g1VJY+yI8M3fozy9J0hbKgNNfz9wPf/wm//Dk+9k7PcH1f1w4tOdfsQju+TfY/Tgg4Ilf\nDe35JUnaghlw+uvVr4Fzf05jqcQto77AS7//Dh1dlaE7/6/+F6QKvOXrsOMBMN+AI0lSlQFnc0w+\nGM7/DSsnHcrnyt/gmR+cD13tg3/exffBgz+CIz4A2+wCexwPi2ZB24rBP7ckScOAAWdzjd2WCef9\njOua/pZdF/wnfOckWLF48M6XEtzxWRjzKnjdP2brdj8eUhme/M3gnVeSpGHEgDMASo2NtB9zMX/f\n8THKzz4K/3E0/OW3fTtISvDwT2DlXzdc7pGfwcJ74LjPQstW2bqdD4Pm8c7DkSQpZ8AZIO9sncJv\nG4/g/+5yFYyZCN//G3jolk3buVKB2z4O/3kWXPlaeOy/11+uqx1+8TnYfh846Mzu9Q1NMPVomH9n\nFpQkSRrhDDgDZKuWJt5x8GS+/WgTL5z23zDlMLj57+D+6za8Y6UCt/0DzP42tJ4LW02GH74nCzyd\na3uW/eM34cW/wJu+BA2NPbftcRysWAjPPzGwX0ySpGHIgDOAzjpiVzq6KvzwweVwxs2w2+vhpx/I\ngsn6VCrws4/AnO/CUR+HN38V3v9LOOJDMOtbcPXr4a8PZWVXPw+/uQz2eGM2qbi33fN1DlNJkmTA\nGUjTJo3nyD225bp7n6KroQVO+xHs9Wa4/RNw99d7Fq6U4dYPwf3fh6M/BcddDBHQOApOuBTOuAXW\nvADfPA7uvQp+/WXoWJX13qzPxKkwcTcvF5ckCQPOgDvriF15ZkUbv3xkaRZW3n0t7PdO+OXn4c5L\nszkylTL85AMw9zp4/T9lE4Yjeh5oj+PhwnuyXqCffxpmfRMOORu2n/7KJ9/9eFjw26G5VF2SpC1Y\n48aLqC+O33sSkyeM5uq7nuSwqdsycWwzvONqaBoNd10GHath9TKYdyMc+89wzCdf+WDjtoP33pAN\nVz38Ezj2Mxs++R7HZ0Fo4b2w2zED+8UkSRpGDDgDrKEUXPj63fnnnzzEa/73Lzl++iTeecgUjnnz\n12lqHgv3XpEVPP5z2bybjYmAw87LXhuz61FQasrm4RhwJEkjmAFnEJxx+C4csss23DxnET+Zu5if\nP/xXXjWumVMOOJPzXrsDO2w7EQ45a5OP19ZZ5vGlK9l/yoQNFxw1Dl59eHa5+Bu/uJnfQpKk4cs5\nOINk7x234p/fsg+//6fj+daZrbTuMpHv3fsUh9+5J2c/uDcr1nRu0nFeWN3Be795L2/799/xw015\noOfux8HSebBy6WZ+A0mShi8DziBraijxhn0mcdX7DuEPn3kDnz5xOr+b/xx/843fMf/ZlRvcd8Fz\nq3nHN37Hw8+8xH6Tt+JzP32I2Qte2PAJq5eQP3HnAH0DSZKGHwPOEJo4tpkLX7871593OC+t7eTt\nV9zDzEefXW/Z+xa+yDuuvIcVazu5/rzXcN3fHc7kCaO54Af3sWTF2vXuA8CkGdlzqrwfjiRpBDPg\n1MGhu07k1g+/jp0njuHca2fxH795glTziIWfP/RXTrv6Xsa3NHLLB47kkF0msvWYJr55ZitrO7r4\n++/Poa2zvP6Dl0rZMNUTd2Y3EpQkaQQy4NTJ5AmjuenCIzh5vx358n8/yj/e+ABtnWW+87u/cOF1\nc9h7x6245cLXMvVVY9ftM23SeL7+noN4cNEK/umWeT1CUQ97HA9rnoe/PjBE30aSpC2LV1HV0Zjm\nRv79vQcx/c7x/OsvHufeJ59nyYo2Tth3El8/9SBGNze8bJ837jOJf3zjnnz1F4+z705b8f6jdnv5\ngXc/Lnuf/yvY6aBB/haSJG157MGps4jgw8dP46ozDmF1exfnHjmVb5x+yHrDTdWHjt2DE/fdgf99\n+yP89s/LXl5g3PawwwwnGkuSRiwDzhbixP12YO7n3sTn3roPDaXYYNlSKfjXdx/AtO3H86Hr7+ep\n51e/vNDux8PTf4D2DV+pJUlSERlwtiCljQSbWmNHNfLNM1uJgHO/O4sXV3f0LLDH8VDpgj//zwDX\nUpKkLZ8BZxh79bZjuOqMQ3j6xbWc/d1ZrG7v6t648+EwbhLcdC5852S4/wfQvqp+lZUkaQgZcIa5\nw3fbliveezAPLV7BBT+YQ3tXfvl4YzP8/V3ZM69WLYWffhD+757ZU8wX3J091VySpIKKV7zUuEBa\nW1vT7Nmz612NQXXTnEV84j8f4M0zduTy0w7qOY8nJXj6jzD3B/DQj6FjJWz9atjpAHjVntlr22nw\nqj2gZev6fQlJkjYiIuaklFo3Vs7LxAvinYdMYfmaDr502yNsNbqJ//32/YjIQ04EvPo12evEf4FH\nfgaP3ArPPgqP/Xc2V6dq3A7wqmlZ6Nlur/x9OozfITuOJEnDgAGnQN5/1G68sLqDb/z6CSaObeKT\nJ0x/eaHmMXDAqdkLoNwJLy6A5x6H5/6cvx6DeTdB+4ru/UZtlYWdMdtClKDUkAWeaMg+NzTB2O1g\n/I6w1Y6w1eRsefyO2XCZJElDyIBTMJ88YS9eXNPJFTOfYJsxzeu/EWCthqa8x2Zaz/UpZXN3lj2W\nhZ9lj2XBZ9VSSOVse6WcL1eg3AGrlkHXep6T1bI1NDRDqRFKTVk4KjVmr60nw6uPgF1eCzsdDE0t\nA9cYkqQRy4BTMBHBl/5mP1aszYarnlnexjlH7srOE8f09UDZsNT4HWC3YzZtn5SgbTm8tARWPgMv\nPZMtr3kuGwardEE5f690Zr1Hzz8Bd/6vbP+GUTD5kCzsvPqIrCeoaTQ0jc3em8dm4UiSpI1wknFB\ntXeV+eyPH+LH9y+mkhLHT5/EWa/dhdft8aruuTlbijUvwMLfw1P3ZK8lD2Q9Q+vT0AxNY7Kw0zS6\nZnlM9rmxpbuHaN1QWkP3e5Sy8FaqLpeAgHI7dHVAVxt0tXe/k2DU+F6vrbL3UmN3cKt0ZT1a1feG\n5rx+eZ2aWqCxWr9SzzpV6wJZb1i1Z6yS946lSrZf81hoHgejxmXH39L+HCVpCGzqJGMDTsEtWbGW\n6+5dyA//uJDnV3ew+3ZjOeu1u/KOg6cwbtQW2oHXvgqeuS8LPp1roXN19t6xJlvuWNNr/WroXJOt\nL7fnQaM2JFTfU/eQWm14SBVoHJWFiMaWmuVRWX06VmV3hG57KTv+lqDU2B3sUqU7WPVeJuW3BKh5\nByBeHrCiIQtfKS+7bv9K920FGpuznraG5u7lde/V9S09t607Xk2b927/6vmq52xohjETYfTE7H3M\ntt3LkAfQjjyY5q9UzoZDR0+E0dvk+2+ThczeqkOslc7sWJ1re73WZMOupcbu71P9fVS/Y0Njtlwd\nfjVwSkPCgFNjJAecqrbOMrfPW8K19yzggUUrGN3UwF47jGeP7ccxbftx+ft4Jm8zeqOPihjRutqz\nANa+IvtLsnY+UamxOyyUO7v/suxaC51t3X9pvix81fzFXg0ZvXudutqyoNWxuvu9fVV2zNqg0qPn\nKu+dinj5e21PUarkdcjrVO3ZiujuWYpSVr91oaIj+y7rlttfHjiq62ono5dKNccv1fSq1ZwvImuv\ntS9kIbfSuXl/Zo15T1qlK/tzqXT2vHJwoJSaauaaVf8MG3v++TTUlKmGo4b8t1P7u1i3XBtc1/O7\nWRdc6bkcJWjZKgt4o7eB0ROgZUK2PGr8y3+31fpVA3K1ncpd+W+2q+b3VfNbq/3NVX9btX++qdJd\n10pX93dKlay91vVu1vzDoqEp/zqVLI/XBu1SY3cvZvNYaB6ftV9vlUr3b7G8gd9PlLLzNo0e/sPf\nl10Ghx4Kxx7bvW7mTJg1Cz71qfrVaxAYcGoYcHq6f+GL/HTuMzy+dCV/fnYVy1Z290qMaiyx+3bj\nmDapGnyyELTLtmNoavC+kBpiKWWBbs0L3YEnorvHqHFUd09RNEDbiu5ya1/sXu5qywNI9S/1puwv\n0moPTXW4s6kle6/+hVvp6u4tqg5b1v7FWe7onk9W7siDQVevgFLpXq5uL1cD4isFiFLPMLG+4Fs7\ntEnUvOUBtm0FrF2et0P+Wt9FAMNddfg2pawtu9r7F4rX9da1vLwXskcP5aiePY7VEFfb0/myf1SU\nNqGHL2rK5MulxmxIvGXrPKRunb8mZEF13RD4aPj9fXDOhXD9D+B1h8Mv74CzL4Qr/hccPA3aX8r+\nW4KeIZSaf8jUfpfaXtaG5qyNR42vCZj5K2I9PbL5a5tdByU4GnBqGHA2bMWaTuYvW8n8Z1cx/9lV\nPL40e1+8vPt/hk0NwdRXjWXa9uPZc9J49pw0jmmTxrPrtmNoNPhIw0Nn3hPYI4DVzB+LPFg1NHX3\nSFVDYe3wZ6V237w3rPdQY6r2BlZ7e3qFtnI+PNhj3ltbd69fbTCofq505UPGq2reV2Y9mkSvYFIT\nSl5JqnSfv3Ntr3q0v0IPZUcWJHv0Pua9pgTrHRZOlQ3/udQOH9fuWylnwaRtRfa+MX/pgpvWQmsT\nzO6Ed46GqXWcivDpp7JgNsC80Z822dZjmjhkl4kcssvEHutXt3fxxLJV/HnpKv787Cr+vHQl8xav\n4PaHlqz7h0pzQ4ndthvLHtuPY9JWLWw7rpltxzYzcewoJo7Nlrcd18y4UY1b3uRmaaRpavFWDMNV\nuas77LStyOYFduVD351t+VD4Wogfw7W/gPefAh87O+t1adm6u/elOkTdew4c9AxrtUPI5XxovmN1\nFiirAXNdj1CvsFd9rW/+2xAy4OgVjR3VyP5TJrD/lJ4JfG1HOe/pWcnjz67kz0tX8eCiFTy36lnW\ndKz/6qcxzQ1sP34U22/VwvbjRzEpf3/1xDHsveNWvHrimD49TV2SRpSGxnzC/cRXLjNzJtx2P1x8\nMVx5Jbz3oz3n5IwwBhz12ejmBmZM2ZoZU17+3Kq2zjLPr+7g+VXtPL+6gxdWdfD86naefamdpSvb\nWfpSGw8/8xK/euRZ1nZ2h6Exzdmk57133Cp77TCe7ce3MK6lkXGjGmludBhMkl7RzJnw7nfDjTdm\noebYY3t+HoEMOBpQLU0NTJ4wmskTNtw1mVJiZXsXf1m2mkf/+hKPLFnJn5a8xM8eeIbr/7DwZeWb\nG0uMH9XIuJZGxjY30tJUYlRjw7r3UU0lRjVmy40NQXNDiabqq7Hn5+bGEk15mWy5RGNDZO+lWPe5\nsZSVK+VDa6VSUAooRRCRDc+1NDUwqrHk8Juk+po1q2eYOfbY7POsWSM24DjJWFuUlBLPrGjj8b+u\n5IXVHaxq72JVexcr27pY1d7Jqrbsc3tXhfbOCm1d5R7v7V1lusqJjnKFznKFyhD8vCOyq89GNzUw\nuqmBlqYGSqWgIWJdKGooBRGRB6gsRI1q7A5Y1bDV0tTQI7xVA1RDqURDKQtXDfmxI18uVa/OJfL5\nmEEAjQ2xrj6jmxoY3dxAS2MDLc3Z+QxlkoYjJxlrWIqITeoB2lTlSqKzXKGjXKGrnC93VdYFoM6u\nREe5TGc5rdveWa7Qle/XVU4koJISKaXswoYE5ZTo7MqCVVtHmbauCms7yrR1llnbWaaSEuVKopKg\nUknZ5wTlSnbOlZ1dPN9VWVe3jq7s1daZHas8BMmssRTreq6yV9ZrVe3hynqySjTXLDfl+3Qvd5et\nhrRRjQ005+GtuTEr11Ddr5T1kjW87NzZ/o01x2pqzLeXupcb86AoSRtjwFGhNZSChlLWizGcdJUr\ntHVVaK8GnnKivC40db9XKpDIglRKWRirBrHOcloXwNZ2lmnrrOTvZdq7KnTlQa6jq0JXpbKu56s2\n6HWUsyDXWa6wuqOc7VNOdOblu8oVOis1wbErO+Zgaq6Gqprer6basNTYHcq6hyV7fh7V2HOfLJh1\n96pVe9aqPWWlvGesOkTZUIqanrEs1I1uzj43NxrEpC2BAUfaAjU2lBjXUNpyH6exAeU8NHV0VWgv\nlylXst6xrkqiXMkCUPVzV7nam9YdpKqfu2pDVjkLZB1deaDq6rmtGrA6az5XQ1lnV7UHr9p713Of\nwQpkpcj+HKs9Vk3r6e2qnRNWDVuNNWUbG0q0NJUY29zI2FHZa9yohnXL2VBjd+iqBrGGiB7BrtrT\nVtvL5h3LVXTD7/+ekrZoDaXIejOaG4CmeldnoyqVrOeqvat7+LKjqzqHK+slq6wbnsx6ybrKlR49\nYuveO8rrhjhrQ11nOXX3kvU6R2e+blVXV4/es2rIa+uqsKq9i46ujdwsro8aS5H1WjU10NxQ6jFR\nf93k/cZsfXNNSGoo1Qw1NmRhCrqfclY7rbOpIXrMAVs3J6y5Ie9F6zUXrTEPfzXBzxuJqr8MOJJG\ntFIpaBkGw5id5Qpr2sus6uhidT75vqucD1XWBLHqEGa1N6urUp1rVukxlNg9MT+bnN/Wmfe6dWVD\nmGs7yyxf27GuTHVIszqvrVxJ6z5XR+OixyMjsjpv7nUspWBd+KkO/QX5ZPrITlWKoKkxukNZzVyw\nUY3dk/V7vzc3dge3xlL16snI5571DH299+vdI+d9vLY8BhxJGgaaGkpsPabE1mO2/F6xqpSyYNXW\nUWFNZxdr8/lgazvKPXqyeoevjpoLAmovDMgm/We9abXzzSop6zFrrwlo7Z0Vlq/poK2ze111/llb\nZ3nAr7CsTtov5QGsejuJUql6VWPPsFUbwhpLL7/isiEfdqz2ZlVf6+aJ1d4KoyG6b3lRU4/qFZel\nfJi0oRTZlZRNpXVXVVZ704o4Z8yAI0kaFBHVXpUGtt6ChitTynqf2vMJ/NVJ8909U9Wercq6209U\nP7d1lnsMJbZ3VXoMK6ZUnfTfffVl9QKB2h6z2hDW9bIrLhOVvIesZ/DLzjEYmhtL2Rwuet7vq1TK\ne7VK67+ysXqLjJYer2zdh47bgzHN9YsZBhxJ0ogS0X0/quGmds5Y7ZWQnTXL5UrPkFTJP3flIavH\nvLH8Csv2rjKk6jBnd0DrHvKs9LjHWHW5vbPCc6s6eh4rX77w9bvXta0MOJIkDRPDZc7YlmD4xVdJ\nkqSNMOBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTC\nMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCMeBIkqTCqUvAiYgTI+KxiJgfERet\nZ3tExOX59gcj4uBN3VeSJGnIA05ENABXACcB+wCnRcQ+vYqdBEzLX+cDV/ZhX0mSNMLVowfnMGB+\nSunJlFIH8CPglF5lTgG+lzL3AhMiYsdN3FeSJI1wjXU452Tg6ZrPi4DXbEKZyZu4LwARcT5Z7w/A\nqoh4bDPqvCGvAp4bpGMPN7ZFN9uim23RzbboZlt0sy26bUpb7LIpB6pHwBkSKaWrgasH+zwRMTul\n1DrY5xkObItutkU326KbbdHNtuhmW3QbyLaoR8BZDOxc83lKvm5TyjRtwr6SJGmEq8ccnFnAtIiY\nGhHNwHuAW3uVuRU4M7+a6nBgRUppySbuK0mSRrgh78FJKXVFxIeAO4AG4JqU0sMRcUG+/SrgduBk\nYD6wBjhnQ/sO9XfoZdCHwYYR26KbbdHNtuhmW3SzLbrZFt0GrC0ipTRQx5IkSdoieCdjSZJUOAYc\nSZJUOAacfhrJj4yIiGsi4tmIeKhm3cSI+EVE/Dl/36aedRwqEbFzRMyMiD9FxMMR8dF8/Yhrj4ho\niYg/RsQDeVt8IV8/4tqiKiIaIuL+iPiv/POIbIuIWBAR8yJibkTMzteN1LaYEBE3RcSjEfFIRBwx\nEtsiIvbKfw/V10sR8bGBbAsDTj/4yAi+C5zYa91FwK9SStOAX+WfR4Iu4OMppX2Aw4EP5r+Fkdge\n7cBxKaUDgAOBE/OrIEdiW1R9FHik5vNIbotjU0oH1tzjZKS2xf8Dfp5Smg4cQPb7GHFtkVJ6LP89\nHAgcQnZB0Y8ZwLYw4PTPiH5kRErpLuCFXqtPAa7Nl68F/mZIK1UnKaUlKaX78uWVZP+zmswIbI/8\n0Sqr8o9N+SsxAtsCICKmAG8GvlWzekS2xSsYcW0REVsDRwPfBkgpdaSUljMC26KX44EnUkpPMYBt\nYcDpn1d6lMRINim/VxHAX4FJ9axMPUTErsBBwB8Yoe2RD8nMBZ4FfpFSGrFtAXwd+BRQqVk3Utsi\nAb+MiDn5Y3RgZLbFVGAZ8J186PJbETGWkdkWtd4D/DBfHrC2MOBowKXs3gMj6v4DETEOuBn4WErp\npdptI6k9UkrlvMt5CnBYROzXa/uIaIuIeAvwbEppziuVGSltkXtd/rs4iWwY9+jajSOoLRqBg4Er\nU0oHAavpNQQzgtoCgPymvW8D/rP3ts1tCwNO/2zK4yZGmqX5E9/J35+tc32GTEQ0kYWb61JKt+Sr\nR2x7AOTd7jPJ5mqNxLY4EnhbRCwgG8I+LiJ+wMhsC1JKi/P3Z8nmWRzGyGyLRcCivGcT4CaywDMS\n26LqJOC+lNLS/POAtYUBp398ZMTL3QqclS+fBfy0jnUZMhERZOPpj6SUvlqzacS1R0RsFxET8uXR\nwBuBRxmBbZFS+qeU0pSU0q5k/3+4M6V0BiOwLSJibESMry4DbwIeYgS2RUrpr8DTEbFXvup44E+M\nwLaocRrdw1MwgG3hnYz7KSJOJhtjrz4y4tI6V2nIRMQPgdeTPdZ+KfB54CfAjcCrgaeAd6eUek9E\nLpyIeB3wW2Ae3XMtPkM2D2dEtUdE7E82KbCB7B9PN6aUvhgR2zLC2qJWRLwe+ERK6S0jsS0iYjey\nXhvIhmiuTyldOhLbAiAiDiSbeN4MPEn2KKISI7MtxgILgd1SSivydQP2uzDgSJKkwnGISpIkFY4B\nR5IkFY4BR5IkFY4BR5IkFY4BR5IkFY4BR9Kg+//t3V2IFWUcx/HvDxfqKiNBLBS2iJAEoxdCJPZi\nicCoCysiwougooukF8LoprQS7QW6KZXFu16ooJe98KK0IkjYNgIlLNgKkhI3LUOLYJFdf13MszQd\nz9Zmu+ecxt8Hhpn/zH+eec65OPyZeeY8kjZJ8gzLui70x5LWd/q6EdE5fd3uQEScNU5w+iz0AN92\nuiMR0XwpcCKiUyZtf9rtTkTE2SGPqCKi6yT1l8dGd0p6RdJvko5K2tgmd1DSqKQJSUckbS+TndZz\nFkkakjRe8sYkPdTS1AJJWyT9VK61TdI5tTbOL7M9Hy5tfC9p5zx9BRExx3IHJyI6RtJpvzm2J2vh\n88Au4DZgANgo6Wfb28r5K4D3gD3ArVST3j4DXEJ5/FXmwfoYWAw8STUf1qVlqXsE+AhYB6wEtlL9\nNfxz5fgLwGrgYeDHcq0BIuJ/IVM1RMS8k7SJas6ydi4u6++APbZvqJ23E7gRWGb7lKQ3gKuB5ban\nSs7twJvAatsjku4DdgBX2d4/Q38MfGJ7oLZvGFhie1WJDwBDtl88088dEd2TOzgR0SkngOvb7D8M\nXFS232059g5wD7CUalK+a4G3poub4m1gErgOGAEGgX0zFTc1u1vir4BravF+YIOkKeAD21//Q3sR\n0UMyBiciOmXS9udtlpO1nKMt50zHF9bWR+oJpdg5BlxQdi0CxmfRn+Mt8Ung3Fq8HhgGngDGJH0j\n6Y5ZtBsRPSAFTkT0ksUzxOO19V9yJC2gKmp+KbuO8WdBdMZsH7f9gO0lwBXAKPCapMv/a9sRMf9S\n4EREL1nbEt9CVdQcKvEosLYUNfWcPmBviT8ErpS0cq46ZfsLYAPVb+byuWo3IuZPxuBERKf0SVrV\nZv8Pte0VkoaoxtUMAHcDD9o+VY5vBvYBw5J2UI3NeRZ43/ZIyXkZuB/YXQY3j1ENZL7M9mOz7ayk\nvVRjgg4ABu4Ffgc+m20bEdE9KXAiolMWUg0CbvU48GrZfhS4iarAmQCeBl6aTrT9paQ1wBaqAci/\nAq+X86ZzJiQNUr0+/hRwHnAQ2P4v+zsC3AX0A1NUhdUa24f+5pyI6BF5TTwiuk5SP9Vr4jfb3tXd\n3kREE2QMTkRERDROCpyIiIhonDyiioiIiMbJHZyIiIhonBQ4ERER0TgpcCIiIqJxUuBERERE46TA\niYiIiMb5Ay6tZE9Cjs2RAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x19199a358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 8))\n",
    "plt.title(\"Learning curve\")\n",
    "plt.plot(results.history[\"loss\"], label=\"loss\")\n",
    "plt.plot(results.history[\"val_loss\"], label=\"val_loss\")\n",
    "plt.plot( np.argmin(results.history[\"val_loss\"]), np.min(results.history[\"val_loss\"]), marker=\"x\", color=\"r\", label=\"best model\")\n",
    "# plt.xlim([30,50])\n",
    "plt.ylim([0,0.05])\n",
    "# plt.rc('font', weight='bold')\n",
    "# plt.yscale('log')\n",
    "plt.rc('xtick.major', size=5, pad=7)\n",
    "plt.rc('ytick.major', size=5, pad=7)\n",
    "# plt.rc('xtick', labelsize=15)\n",
    "plt.xlabel(\"Epochs\", size=15)\n",
    "plt.ylabel(\"log_loss\", size=15)\n",
    "\n",
    "plt.legend();\n",
    "plt.tight_layout()\n",
    "LossFigDir = '/Users/wtrainor/Library/Mobile Documents/com~apple~CloudDocs/Students/Samir/Geophysics/UNet/ArticleE/Fig/'\n",
    "LossFigName = 'LossFunction'+source_string+'_'+receiver_string+'_100epoch.pdf'\n",
    "plt.savefig(LossFigDir+LossFigName,dpi=800, bbox_inches='tight', transparent=True) #,format='ps')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt('images_for_presentation_paper/'+source_string+'_'+receiver_string+'_Loss_100epochs.csv', np.asarray([results.history[\"loss\"],results.history[\"val_loss\"]]).T,\n",
    "          delimiter=',', header='TrainingLoss, ValidationLoss', fmt='%f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9070352337950824\n"
     ]
    }
   ],
   "source": [
    "print(np.average(f1_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8895980644633187\n"
     ]
    }
   ],
   "source": [
    "print(np.average(recall_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9253178751717619\n"
     ]
    }
   ],
   "source": [
    "print(np.average(precision_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6786 8572 8506]\n"
     ]
    }
   ],
   "source": [
    "print(c_matrices_stacked[:,0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt('images_for_presentation_paper/'+source_string+'_'+receiver_string+'_CV_metrics_100epochs.csv', np.asarray([np.arange(3), eval_stats[:,0], eval_stats[:,1], \n",
    "                                                  c_matrices_stacked[:,0,0], c_matrices_stacked[:,0,1],\n",
    "                                                  c_matrices_stacked[:,1,0],c_matrices_stacked[:,1,1], \n",
    "                                                  f1_stats, recall_stats, precision_stats]).T,\n",
    "          delimiter=',', header='Fold, Loss, Accuracy, True Negatives, False Positives, False Negatives, True Positives, F1-score'\n",
    "          ', Recall, Precision', fmt='%f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}