random_search_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Search Example\n",
    "\n",
    "In this example we will perform a RandomSearch using Scikit-learn and Keras on the [breast cancer](https://github.com/autonomio/datasets/blob/master/autonomio-datasets/breast_cancer.csv) classification task. You can run this example on CPU. It will take more or less 5 minutes.\n",
    "\n",
    "We will continue to use the same example of the [Grid Search](./grid_search_example.ipynb) notebook, but we will search for a different subset of hyperparameters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Setup\n",
    "\n",
    "Import the packages we need for the computation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import wrangle as wr\n",
    "\n",
    "from numpy import nan\n",
    "\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# Mounting point\n",
    "MP = '/floyd/input/bcds'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the dataset\n",
    "\n",
    "Load, clean and preprocess the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def breast_cancer():\n",
    "    '''Load and preprocess(cleaning) the dataset'''\n",
    "    df = pd.read_csv(os.path.join(MP, 'breast_cancer.csv'))\n",
    "    \n",
    "    # then some minimal data cleanup\n",
    "    df.drop(\"Unnamed: 32\", axis=1, inplace=True)\n",
    "    df.drop(\"id\", axis=1, inplace=True)\n",
    "\n",
    "    # separate to x and y\n",
    "    y = df.diagnosis.values\n",
    "    x = df.drop('diagnosis', axis=1).values\n",
    "\n",
    "    # convert the string labels to binary\n",
    "    y = (y == 'M').astype(int)\n",
    "\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "x, y = breast_cancer()\n",
    "\n",
    "# Normalize every feature to mean 0, std 1\n",
    "x = wr.mean_zero(pd.DataFrame(x)).values\n",
    "\n",
    "input_dim = x.shape[1] # number of columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model definition\n",
    "\n",
    "Define the model and the variables to search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras import optimizers\n",
    "\n",
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(first_neuron=9,\n",
    "                 activation='relu',\n",
    "                 kernel_initializer='uniform',\n",
    "                 dropout_rate=0,\n",
    "                 optimizer='adam'):\n",
    "    \n",
    "    # Create model\n",
    "    model = Sequential()\n",
    "    # L1\n",
    "    model.add(Dense(first_neuron, \n",
    "                    input_dim=input_dim, \n",
    "                    kernel_initializer=kernel_initializer, \n",
    "                    activation=activation))\n",
    "    # Dropout\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    # L2\n",
    "    model.add(Dense(1, kernel_initializer=kernel_initializer, activation='sigmoid'))\n",
    "    \n",
    "    # Compile model\n",
    "    model.compile(loss='binary_crossentropy', \n",
    "                  optimizer=optimizer, \n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "model = KerasClassifier(build_fn=create_model) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Range of Values - The Grid\n",
    "\n",
    "Defining the parameter space boundaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify parameters and distributions to sample from\n",
    "\n",
    "# Model Design Components\n",
    "first_neurons = sp_randint(8, 15)\n",
    "activation =  ['relu', 'elu'] # You can also try 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'\n",
    "kernel_initializer = ['uniform', 'normal'] # You can also try lecun_uniform', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'\n",
    "optimizer= ['adam', 'nadam', 'sgd']\n",
    "\n",
    "# Hyperparameters\n",
    "epochs = [10] # You can also try 20, 30, 40, etc...\n",
    "batch_size = [1024] # You can also try 2, 4, 8, 16, 32, 64, 128 etc...\n",
    "dropout_rate = uniform(0.0, 1) # Sample a value of dropout in the range [0, 0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare for the Search\n",
    "param_dist = dict(epochs=epochs, \n",
    "                  batch_size=batch_size, \n",
    "                  optimizer=optimizer,\n",
    "                  dropout_rate=dropout_rate,\n",
    "                  activation=activation,\n",
    "                  kernel_initializer=kernel_initializer,\n",
    "                  first_neuron=first_neurons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid Search\n",
    "\n",
    "Run the search: 3 folds cross-validation (`cv=3`) on a single process (`n_jobs=1`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 877us/step - loss: 0.6857 - acc: 0.6385\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6849 - acc: 0.6016\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6776 - acc: 0.7098\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6761 - acc: 0.7230\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6727 - acc: 0.7388\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6697 - acc: 0.8047\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6667 - acc: 0.8047\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6629 - acc: 0.8338\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6543 - acc: 0.8734\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6553 - acc: 0.8628\n",
      "190/190 [==============================] - 0s 213us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam, total=   1.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 650us/step - loss: 0.7021 - acc: 0.3773\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6987 - acc: 0.4327\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6939 - acc: 0.4538\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6944 - acc: 0.5145\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6879 - acc: 0.6095\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6848 - acc: 0.6253\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6844 - acc: 0.6412\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6787 - acc: 0.6992\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6726 - acc: 0.7573\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6733 - acc: 0.7599\n",
      "190/190 [==============================] - 0s 218us/step\n",
      "379/379 [==============================] - 0s 10us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam, total=   0.8s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 654us/step - loss: 0.6965 - acc: 0.4316\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6945 - acc: 0.4789\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6919 - acc: 0.5342\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6909 - acc: 0.5500\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6876 - acc: 0.6316\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6856 - acc: 0.6895\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6830 - acc: 0.7289\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6820 - acc: 0.7395\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6778 - acc: 0.8132\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6755 - acc: 0.7921\n",
      "189/189 [==============================] - 0s 303us/step\n",
      "380/380 [==============================] - 0s 3us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.6010381533043048, epochs=10, first_neuron=10, kernel_initializer=normal, optimizer=adam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 718us/step - loss: 0.6912 - acc: 0.7309\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6900 - acc: 0.7678\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6891 - acc: 0.7678\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6883 - acc: 0.7652\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6869 - acc: 0.7573\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6853 - acc: 0.7836\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6848 - acc: 0.7784\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6833 - acc: 0.7731\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6820 - acc: 0.7599\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6809 - acc: 0.7757\n",
      "190/190 [==============================] - 0s 310us/step\n",
      "379/379 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam, total=   0.8s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 770us/step - loss: 0.6929 - acc: 0.5409\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6917 - acc: 0.6412\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6902 - acc: 0.7256\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6890 - acc: 0.7520\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6882 - acc: 0.7889\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6868 - acc: 0.8021\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6865 - acc: 0.8285\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6853 - acc: 0.8575\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6849 - acc: 0.8443\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6825 - acc: 0.8522\n",
      "190/190 [==============================] - 0s 374us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 830us/step - loss: 0.6936 - acc: 0.4579\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6918 - acc: 0.5421\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6908 - acc: 0.6474\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6897 - acc: 0.7105\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6888 - acc: 0.7395\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6880 - acc: 0.7789\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6866 - acc: 0.8053\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6858 - acc: 0.8184\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6851 - acc: 0.8184\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6840 - acc: 0.8184\n",
      "189/189 [==============================] - 0s 413us/step\n",
      "380/380 [==============================] - 0s 11us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.39983996327340543, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=adam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 895us/step - loss: 0.6901 - acc: 0.5805\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6877 - acc: 0.6464\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6857 - acc: 0.7018\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6847 - acc: 0.7098\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6811 - acc: 0.7361\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6776 - acc: 0.7731\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6732 - acc: 0.7784\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6732 - acc: 0.7836\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6723 - acc: 0.7916\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6710 - acc: 0.7757\n",
      "190/190 [==============================] - 0s 503us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 951us/step - loss: 0.6967 - acc: 0.4723\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6985 - acc: 0.4433\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6947 - acc: 0.5356\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6929 - acc: 0.5594\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6908 - acc: 0.5963\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6884 - acc: 0.6385\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6864 - acc: 0.6570\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6849 - acc: 0.6807\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6837 - acc: 0.7124\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6826 - acc: 0.6966\n",
      "190/190 [==============================] - 0s 528us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam, total=   1.1s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 998us/step - loss: 0.6964 - acc: 0.5105\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6951 - acc: 0.5474\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6932 - acc: 0.5500\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6906 - acc: 0.5711\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6899 - acc: 0.5816\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6880 - acc: 0.6079\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6858 - acc: 0.6237\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6844 - acc: 0.6737\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6809 - acc: 0.6868\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6801 - acc: 0.7000\n",
      "189/189 [==============================] - 0s 623us/step\n",
      "380/380 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.7129907024046704, epochs=10, first_neuron=13, kernel_initializer=normal, optimizer=adam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 987us/step - loss: 0.6946 - acc: 0.3931\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6941 - acc: 0.5066\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6937 - acc: 0.5884\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6933 - acc: 0.6253\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6929 - acc: 0.6702\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6920 - acc: 0.6807\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6917 - acc: 0.6887\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6912 - acc: 0.7045\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6906 - acc: 0.6966\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6903 - acc: 0.7018\n",
      "190/190 [==============================] - 0s 604us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6912 - acc: 0.7203\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6909 - acc: 0.7177\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6913 - acc: 0.6755\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6906 - acc: 0.7018\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6907 - acc: 0.6913\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6902 - acc: 0.6992\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6899 - acc: 0.6913\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6897 - acc: 0.6807\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6896 - acc: 0.6807\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6894 - acc: 0.6649\n",
      "190/190 [==============================] - 0s 712us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 1ms/step - loss: 0.6938 - acc: 0.4368\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6936 - acc: 0.4658\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6935 - acc: 0.4474\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6935 - acc: 0.4447\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6934 - acc: 0.4474\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6935 - acc: 0.4763\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6930 - acc: 0.4816\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6932 - acc: 0.4605\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6929 - acc: 0.4711\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6927 - acc: 0.5053\n",
      "189/189 [==============================] - 0s 812us/step\n",
      "380/380 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.3072453873479509, epochs=10, first_neuron=10, kernel_initializer=uniform, optimizer=sgd, total=   1.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6902 - acc: 0.6412\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6905 - acc: 0.6332\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6902 - acc: 0.6438\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6894 - acc: 0.6675\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6905 - acc: 0.6438\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6884 - acc: 0.6623\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6886 - acc: 0.6887\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6853 - acc: 0.6939\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6857 - acc: 0.6992\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6851 - acc: 0.7071\n",
      "190/190 [==============================] - 0s 797us/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd, total=   1.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6921 - acc: 0.6016\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6903 - acc: 0.6148\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6900 - acc: 0.6385\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6898 - acc: 0.6069\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6888 - acc: 0.6491\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6888 - acc: 0.6332\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6872 - acc: 0.6332\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6893 - acc: 0.6069\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6860 - acc: 0.6174\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6901 - acc: 0.6253\n",
      "190/190 [==============================] - 0s 807us/step\n",
      "379/379 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd, total=   1.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 1ms/step - loss: 0.6954 - acc: 0.4868\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6966 - acc: 0.4711\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6931 - acc: 0.5342\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6896 - acc: 0.5763\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6927 - acc: 0.5263\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6902 - acc: 0.5447\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6933 - acc: 0.5237\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6879 - acc: 0.5947\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6907 - acc: 0.5447\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6886 - acc: 0.5632\n",
      "189/189 [==============================] - 0s 901us/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.9201662691113225, epochs=10, first_neuron=12, kernel_initializer=uniform, optimizer=sgd, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 1ms/step - loss: 0.6927 - acc: 0.5515\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6918 - acc: 0.7203\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6908 - acc: 0.7784\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6898 - acc: 0.8100\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6887 - acc: 0.8338\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6875 - acc: 0.8417\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6865 - acc: 0.8522\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6852 - acc: 0.8707\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6838 - acc: 0.8628\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6827 - acc: 0.8813\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6957 - acc: 0.2559\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6945 - acc: 0.3245\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6933 - acc: 0.4908\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6922 - acc: 0.6412\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6912 - acc: 0.7652\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6903 - acc: 0.8021\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6893 - acc: 0.8179\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6881 - acc: 0.8522\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6870 - acc: 0.8602\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6857 - acc: 0.8628\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam, total=   1.3s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6910 - acc: 0.7500\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6901 - acc: 0.7921\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6892 - acc: 0.8184\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6883 - acc: 0.8263\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6872 - acc: 0.8474\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6864 - acc: 0.8447\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6853 - acc: 0.8684\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6839 - acc: 0.8658\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6832 - acc: 0.8789\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6813 - acc: 0.8789\n",
      "189/189 [==============================] - 0s 1ms/step\n",
      "380/380 [==============================] - 0s 11us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.1246912886611955, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=adam, total=   1.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 1ms/step - loss: 0.6836 - acc: 0.7361\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6827 - acc: 0.7546\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6816 - acc: 0.7889\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6803 - acc: 0.8047\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6794 - acc: 0.8179\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6784 - acc: 0.8100\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6775 - acc: 0.8364\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6763 - acc: 0.8285\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6753 - acc: 0.8338\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6745 - acc: 0.8311\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd, total=   1.2s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6864 - acc: 0.6939\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6861 - acc: 0.7150\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6852 - acc: 0.7177\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6846 - acc: 0.7256\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6840 - acc: 0.7441\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6836 - acc: 0.7573\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6830 - acc: 0.7731\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6816 - acc: 0.7757\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6817 - acc: 0.7625\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6813 - acc: 0.7836\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd, total=   1.4s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6990 - acc: 0.3553\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6984 - acc: 0.3500\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6971 - acc: 0.3974\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6966 - acc: 0.4237\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6953 - acc: 0.4500\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6947 - acc: 0.4868\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6943 - acc: 0.5079\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6934 - acc: 0.5342\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6926 - acc: 0.5474\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6917 - acc: 0.5711\n",
      "189/189 [==============================] - 0s 1ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.06949616977881301, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=sgd, total=   1.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6903 - acc: 0.6649\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6890 - acc: 0.6385\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6879 - acc: 0.7441\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6870 - acc: 0.7520\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6859 - acc: 0.7704\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6858 - acc: 0.7731\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6843 - acc: 0.7836\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6836 - acc: 0.7889\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6828 - acc: 0.8153\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6815 - acc: 0.7889\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 11us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd, total=   1.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6932 - acc: 0.4802\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6930 - acc: 0.5277\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6922 - acc: 0.5805\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6922 - acc: 0.5910\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6919 - acc: 0.6174\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6912 - acc: 0.6227\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6906 - acc: 0.6517\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6904 - acc: 0.6623\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6896 - acc: 0.6781\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6890 - acc: 0.6755\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd, total=   1.4s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6913 - acc: 0.6132\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6906 - acc: 0.6711\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6900 - acc: 0.7158\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6904 - acc: 0.7053\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6894 - acc: 0.7316\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6897 - acc: 0.7553\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6894 - acc: 0.7447\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6888 - acc: 0.7868\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6884 - acc: 0.7737\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6883 - acc: 0.7947\n",
      "189/189 [==============================] - 0s 1ms/step\n",
      "380/380 [==============================] - 0s 11us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.4902729543087778, epochs=10, first_neuron=13, kernel_initializer=uniform, optimizer=sgd, total=   1.4s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:   26.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "569/569 [==============================] - 1s 1ms/step - loss: 0.6953 - acc: 0.4728\n",
      "Epoch 2/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6928 - acc: 0.5009\n",
      "Epoch 3/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6909 - acc: 0.5870\n",
      "Epoch 4/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6880 - acc: 0.6028\n",
      "Epoch 5/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6845 - acc: 0.6538\n",
      "Epoch 6/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6805 - acc: 0.7030\n",
      "Epoch 7/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6787 - acc: 0.7557\n",
      "Epoch 8/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6763 - acc: 0.7399\n",
      "Epoch 9/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6734 - acc: 0.7610\n",
      "Epoch 10/10\n",
      "569/569 [==============================] - 0s 4us/step - loss: 0.6702 - acc: 0.7979\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score='raise',\n",
       "          estimator=<keras.wrappers.scikit_learn.KerasClassifier object at 0x7f79ec3c3c50>,\n",
       "          fit_params=None, iid=True, n_iter=8, n_jobs=1,\n",
       "          param_distributions={'epochs': [10], 'batch_size': [1024], 'optimizer': ['adam', 'nadam', 'sgd'], 'dropout_rate': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f79ec360ac8>, 'activation': ['relu', 'elu'], 'kernel_initializer': ['uniform', 'normal'], 'first_neuron': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f79ec3609b0>},\n",
       "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "          return_train_score='warn', scoring=None, verbose=2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform the Search!\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "# run randomized search\n",
    "n_iter_search = 8 # Use half number of iterations performed by Grid Search\n",
    "random_search = RandomizedSearchCV(estimator=model, \n",
    "                                   param_distributions=param_dist,\n",
    "                                   n_iter=n_iter_search,\n",
    "                                   n_jobs=1, cv=3, verbose=2)\n",
    "random_search.fit(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "\n",
    "Let's see which configuration give us the best performance.\n",
    "\n",
    "*Note*: it's likely that the Random Search we will perform slightly worse than Random Search in this toy example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.894552 using {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.6010381533043048, 'epochs': 10, 'first_neuron': 10, 'kernel_initializer': 'normal', 'optimizer': 'adam'}\n",
      "0.894552 (0.034227) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.6010381533043048, 'epochs': 10, 'first_neuron': 10, 'kernel_initializer': 'normal', 'optimizer': 'adam'}\n",
      "0.806678 (0.184844) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.39983996327340543, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'uniform', 'optimizer': 'adam'}\n",
      "0.843585 (0.077833) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.7129907024046704, 'epochs': 10, 'first_neuron': 13, 'kernel_initializer': 'normal', 'optimizer': 'adam'}\n",
      "0.581722 (0.065330) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.3072453873479509, 'epochs': 10, 'first_neuron': 10, 'kernel_initializer': 'uniform', 'optimizer': 'sgd'}\n",
      "0.773286 (0.112224) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.9201662691113225, 'epochs': 10, 'first_neuron': 12, 'kernel_initializer': 'uniform', 'optimizer': 'sgd'}\n",
      "0.873462 (0.066920) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.1246912886611955, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'adam'}\n",
      "0.745167 (0.054693) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.06949616977881301, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'normal', 'optimizer': 'sgd'}\n",
      "0.801406 (0.106561) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.4902729543087778, 'epochs': 10, 'first_neuron': 13, 'kernel_initializer': 'uniform', 'optimizer': 'sgd'}\n"
     ]
    }
   ],
   "source": [
    "# Show results\n",
    "print(\"Best: %f using %s\" % (random_search.best_score_, random_search.best_params_))\n",
    "means = random_search.cv_results_['mean_test_score']\n",
    "stds = random_search.cv_results_['std_test_score']\n",
    "params = random_search.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**That's all folks - don't forget to shutdown your workspace once you're done 🙂**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}