grid_search_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid Search Example\n",
    "\n",
    "In this example we will perform a GridSearch using Scikit-learn and Keras on the [breast cancer](https://github.com/autonomio/datasets/blob/master/autonomio-datasets/breast_cancer.csv) classification task. You can run this example on CPU. It will take more or less 5 minutes.\n",
    "\n",
    "This example was taken from [Talos](https://github.com/autonomio/talos/blob/master/examples/Hyperparameter%20Optimization%20on%20Keras%20with%20Breast%20Cancer%20Data.ipynb) which is another super interesting library to perfrom Grid and Random Search with Keras."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Setup\n",
    "\n",
    "Import the packages we need for the computation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import wrangle as wr\n",
    "\n",
    "from numpy import nan\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# Mounting point\n",
    "MP = '/floyd/input/bcds'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the dataset\n",
    "\n",
    "Load, clean and preprocess the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def breast_cancer():\n",
    "    '''Load and preprocess(cleaning) the dataset'''\n",
    "    df = pd.read_csv(os.path.join(MP, 'breast_cancer.csv'))\n",
    "    \n",
    "    # then some minimal data cleanup\n",
    "    df.drop(\"Unnamed: 32\", axis=1, inplace=True)\n",
    "    df.drop(\"id\", axis=1, inplace=True)\n",
    "\n",
    "    # separate to x and y\n",
    "    y = df.diagnosis.values\n",
    "    x = df.drop('diagnosis', axis=1).values\n",
    "\n",
    "    # convert the string labels to binary\n",
    "    y = (y == 'M').astype(int)\n",
    "\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "x, y = breast_cancer()\n",
    "\n",
    "# Normalize every feature to mean 0, std 1\n",
    "x = wr.mean_zero(pd.DataFrame(x)).values\n",
    "\n",
    "input_dim = x.shape[1] # number of columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model definition\n",
    "\n",
    "Define the model and the variables to search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "\n",
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(first_neuron=9,\n",
    "                 activation='relu',\n",
    "                 kernel_initializer='uniform',\n",
    "                 dropout_rate=0,\n",
    "                 optimizer='Adam'):\n",
    "    \n",
    "    # Create model\n",
    "    model = Sequential()\n",
    "    # L1\n",
    "    model.add(Dense(first_neuron, \n",
    "                    input_dim=input_dim, \n",
    "                    kernel_initializer=kernel_initializer, \n",
    "                    activation=activation))\n",
    "    # Dropout\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    # L2\n",
    "    model.add(Dense(1, kernel_initializer=kernel_initializer, activation='sigmoid'))\n",
    "    # Compile model\n",
    "    model.compile(loss='binary_crossentropy', \n",
    "                  optimizer=optimizer, \n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "model = KerasClassifier(build_fn=create_model) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Range of Values - The Grid\n",
    "\n",
    "Defining the parameter space boundaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the range of values\n",
    "\n",
    "# Model Design Components\n",
    "first_neurons = [8, 9] \n",
    "activation =  ['relu', 'elu'] # You can also try 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'\n",
    "kernel_initializer = ['uniform', 'normal'] # You can also try lecun_uniform', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'\n",
    "optimizer = ['Adam', 'Nadam'] # You can also try 'SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adamax'\n",
    "\n",
    "# Hyperparameters\n",
    "epochs = [10] # You can also try 20, 30, 40, etc...\n",
    "batch_size = [1024] # You can also try 2, 4, 8, 16, 32, 64, 128 etc...\n",
    "dropout_rate = [0.0] # No dropout, but you can also try 0.1, 0.2 etc..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare the Grid\n",
    "param_grid = dict(epochs=epochs, \n",
    "                  batch_size=batch_size, \n",
    "                  optimizer=optimizer,\n",
    "                  dropout_rate=dropout_rate,\n",
    "                  activation=activation,\n",
    "                  kernel_initializer=kernel_initializer,\n",
    "                  first_neuron=first_neurons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid Search\n",
    "\n",
    "Run the search: 3 folds for cross-validation (`cv=3`) on a single process (`n_jobs=1`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 486us/step - loss: 0.6947 - acc: 0.3193\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6936 - acc: 0.4485\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6925 - acc: 0.5594\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6914 - acc: 0.7177\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6904 - acc: 0.8074\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6893 - acc: 0.8364\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6882 - acc: 0.8575\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6872 - acc: 0.8628\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6861 - acc: 0.8734\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6850 - acc: 0.8786\n",
      "190/190 [==============================] - 0s 146us/step\n",
      "379/379 [==============================] - 0s 5us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   0.7s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 574us/step - loss: 0.6893 - acc: 0.7071\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6882 - acc: 0.7836\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6871 - acc: 0.8364\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6860 - acc: 0.8602\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6848 - acc: 0.8681\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6835 - acc: 0.8813\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6823 - acc: 0.8997\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6810 - acc: 0.9024\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6796 - acc: 0.9103\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6782 - acc: 0.9077\n",
      "190/190 [==============================] - 0s 224us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   0.7s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 618us/step - loss: 0.6910 - acc: 0.7658\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6901 - acc: 0.8079\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6891 - acc: 0.8447\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6880 - acc: 0.8605\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6870 - acc: 0.8684\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 4us/step - loss: 0.6859 - acc: 0.8711\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 4us/step - loss: 0.6847 - acc: 0.8684\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 4us/step - loss: 0.6835 - acc: 0.8684\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6822 - acc: 0.8711\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6808 - acc: 0.8737\n",
      "189/189 [==============================] - 0s 233us/step\n",
      "380/380 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   0.8s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 662us/step - loss: 0.6907 - acc: 0.8997\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6886 - acc: 0.9499\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6869 - acc: 0.9420\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6852 - acc: 0.9340\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6833 - acc: 0.9314\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6811 - acc: 0.9367\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6786 - acc: 0.9314\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6758 - acc: 0.9340\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6726 - acc: 0.9314\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6691 - acc: 0.9340\n",
      "190/190 [==============================] - 0s 314us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   0.8s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 721us/step - loss: 0.6926 - acc: 0.5303\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6906 - acc: 0.6570\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6889 - acc: 0.6755\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6872 - acc: 0.6834\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6854 - acc: 0.6939\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6833 - acc: 0.7361\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6810 - acc: 0.7652\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6784 - acc: 0.7863\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6754 - acc: 0.8100\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6721 - acc: 0.8417\n",
      "190/190 [==============================] - 0s 365us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   0.8s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 750us/step - loss: 0.6952 - acc: 0.2421\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6930 - acc: 0.5447\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6915 - acc: 0.7316\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6899 - acc: 0.8105\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6883 - acc: 0.8237\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6865 - acc: 0.8395\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6845 - acc: 0.8474\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6823 - acc: 0.8632\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6797 - acc: 0.8789\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6769 - acc: 0.8842\n",
      "189/189 [==============================] - 0s 399us/step\n",
      "380/380 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 819us/step - loss: 0.6948 - acc: 0.3087\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6934 - acc: 0.4142\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6921 - acc: 0.5752\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6908 - acc: 0.7150\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6896 - acc: 0.7704\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6884 - acc: 0.8179\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6872 - acc: 0.8522\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6861 - acc: 0.8628\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6849 - acc: 0.8654\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6837 - acc: 0.8707\n",
      "190/190 [==============================] - 0s 444us/step\n",
      "379/379 [==============================] - 0s 8us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   0.8s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6915 - acc: 0.5330\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6900 - acc: 0.6412\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6885 - acc: 0.7414\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6870 - acc: 0.8127\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6855 - acc: 0.8549\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6840 - acc: 0.8575\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6824 - acc: 0.8786\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6808 - acc: 0.8839\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6792 - acc: 0.8945\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6776 - acc: 0.8971\n",
      "190/190 [==============================] - 0s 465us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 938us/step - loss: 0.6873 - acc: 0.6711\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6852 - acc: 0.7763\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 4us/step - loss: 0.6831 - acc: 0.8211\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6810 - acc: 0.8605\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6788 - acc: 0.8868\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6766 - acc: 0.8974\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6745 - acc: 0.8974\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6723 - acc: 0.9026\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6700 - acc: 0.9000\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6678 - acc: 0.9000\n",
      "189/189 [==============================] - 0s 540us/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6932 - acc: 0.4459\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6902 - acc: 0.7441\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6878 - acc: 0.8285\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6854 - acc: 0.8760\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6829 - acc: 0.8971\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6802 - acc: 0.9077\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6773 - acc: 0.9129\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6740 - acc: 0.9182\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6705 - acc: 0.9340\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6667 - acc: 0.9393\n",
      "190/190 [==============================] - 0s 589us/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   0.9s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 1ms/step - loss: 0.6826 - acc: 0.8021\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6792 - acc: 0.8311\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6765 - acc: 0.8443\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6739 - acc: 0.8470\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6711 - acc: 0.8602\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6681 - acc: 0.8602\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6649 - acc: 0.8760\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6614 - acc: 0.8839\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6577 - acc: 0.8865\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6537 - acc: 0.8918\n",
      "190/190 [==============================] - 0s 650us/step\n",
      "379/379 [==============================] - 0s 11us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   1.1s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 1ms/step - loss: 0.6917 - acc: 0.5026\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6889 - acc: 0.6842\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6867 - acc: 0.7895\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6846 - acc: 0.8421\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6824 - acc: 0.8605\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6800 - acc: 0.8605\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6774 - acc: 0.8658\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6745 - acc: 0.8684\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6714 - acc: 0.8868\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6679 - acc: 0.8895\n",
      "189/189 [==============================] - 0s 738us/step\n",
      "380/380 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6942 - acc: 0.4222\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6930 - acc: 0.5858\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6918 - acc: 0.6702\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6905 - acc: 0.7388\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6893 - acc: 0.7863\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6880 - acc: 0.8206\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6867 - acc: 0.8391\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6853 - acc: 0.8575\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6839 - acc: 0.8654\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6824 - acc: 0.8760\n",
      "190/190 [==============================] - 0s 735us/step\n",
      "379/379 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 0s 1ms/step - loss: 0.6929 - acc: 0.4934\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6917 - acc: 0.6042\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6906 - acc: 0.7098\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6894 - acc: 0.7757\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6882 - acc: 0.8047\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6869 - acc: 0.8232\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6856 - acc: 0.8285\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6843 - acc: 0.8443\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6829 - acc: 0.8522\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6814 - acc: 0.8549\n",
      "190/190 [==============================] - 0s 751us/step\n",
      "379/379 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   1.0s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 0s 1ms/step - loss: 0.6933 - acc: 0.5737\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6921 - acc: 0.7211\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6909 - acc: 0.7974\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6897 - acc: 0.8447\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6886 - acc: 0.8579\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6874 - acc: 0.8763\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6862 - acc: 0.8789\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6849 - acc: 0.8763\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6836 - acc: 0.8868\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6823 - acc: 0.8868\n",
      "189/189 [==============================] - 0s 832us/step\n",
      "380/380 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 1ms/step - loss: 0.6946 - acc: 0.6121\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 14us/step - loss: 0.6918 - acc: 0.6755\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6897 - acc: 0.6887\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6877 - acc: 0.7124\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6854 - acc: 0.7493\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6830 - acc: 0.7968\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6803 - acc: 0.8338\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6772 - acc: 0.8496\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6737 - acc: 0.8628\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6699 - acc: 0.8918\n",
      "190/190 [==============================] - 0s 904us/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   1.1s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 1ms/step - loss: 0.6953 - acc: 0.3325\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6929 - acc: 0.5567\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6912 - acc: 0.6675\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6895 - acc: 0.7441\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6879 - acc: 0.7863\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6860 - acc: 0.8100\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6840 - acc: 0.8338\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6816 - acc: 0.8654\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6789 - acc: 0.8786\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6758 - acc: 0.8945\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 7us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6941 - acc: 0.4868\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6914 - acc: 0.6237\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6894 - acc: 0.6868\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6874 - acc: 0.7184\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6854 - acc: 0.7684\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6832 - acc: 0.7947\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6808 - acc: 0.8053\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6782 - acc: 0.8184\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6753 - acc: 0.8316\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6720 - acc: 0.8474\n",
      "189/189 [==============================] - 0s 1ms/step\n",
      "380/380 [==============================] - 0s 5us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6894 - acc: 0.6359\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6878 - acc: 0.7520\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6861 - acc: 0.8259\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6845 - acc: 0.8681\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6828 - acc: 0.8997\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6812 - acc: 0.9235\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6795 - acc: 0.9261\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6778 - acc: 0.9314\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6760 - acc: 0.9367\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6742 - acc: 0.9420\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 3us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   1.2s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6911 - acc: 0.6201\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6899 - acc: 0.7124\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6887 - acc: 0.8074\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6875 - acc: 0.8945\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6863 - acc: 0.9050\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6851 - acc: 0.9077\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6838 - acc: 0.9156\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6825 - acc: 0.9129\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6811 - acc: 0.9182\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6797 - acc: 0.9235\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   1.5s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6955 - acc: 0.4605\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6933 - acc: 0.5421\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6911 - acc: 0.5816\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6889 - acc: 0.6395\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6868 - acc: 0.6974\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6846 - acc: 0.7368\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6824 - acc: 0.7789\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6801 - acc: 0.8132\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6779 - acc: 0.8395\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6756 - acc: 0.8553\n",
      "189/189 [==============================] - 0s 1ms/step\n",
      "380/380 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   1.3s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6937 - acc: 0.4644\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6908 - acc: 0.6913\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6886 - acc: 0.7678\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6864 - acc: 0.8100\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6841 - acc: 0.8522\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6816 - acc: 0.8760\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6788 - acc: 0.8945\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6757 - acc: 0.9050\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6722 - acc: 0.9103\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6685 - acc: 0.9208\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 10us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   1.4s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6981 - acc: 0.3509\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6943 - acc: 0.4565\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6913 - acc: 0.5567\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 4us/step - loss: 0.6885 - acc: 0.6359\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6856 - acc: 0.7414\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6825 - acc: 0.7810\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6793 - acc: 0.8206\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6757 - acc: 0.8443\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6719 - acc: 0.8575\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6679 - acc: 0.8839\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 11us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   1.5s\n",
      "[CV] activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.7001 - acc: 0.2395\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6953 - acc: 0.3658\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6919 - acc: 0.5158\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6888 - acc: 0.6368\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6856 - acc: 0.7316\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6824 - acc: 0.7658\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6790 - acc: 0.7763\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 13us/step - loss: 0.6755 - acc: 0.8132\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6717 - acc: 0.8132\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6678 - acc: 0.8316\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=relu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   1.5s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6915 - acc: 0.7520\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6901 - acc: 0.8311\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6887 - acc: 0.8813\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6872 - acc: 0.8918\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6857 - acc: 0.9050\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6841 - acc: 0.9156\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6825 - acc: 0.9156\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6807 - acc: 0.9182\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6790 - acc: 0.9208\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6771 - acc: 0.9261\n",
      "190/190 [==============================] - 0s 1ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   1.6s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6990 - acc: 0.0739\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6974 - acc: 0.1108\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6959 - acc: 0.2375\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6945 - acc: 0.4222\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6930 - acc: 0.5673\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6915 - acc: 0.6570\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6901 - acc: 0.7124\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6886 - acc: 0.7652\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6871 - acc: 0.7942\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6855 - acc: 0.8285\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   1.5s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6920 - acc: 0.6000\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6904 - acc: 0.6974\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6888 - acc: 0.7816\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6872 - acc: 0.8158\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6855 - acc: 0.8447\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6837 - acc: 0.8579\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6819 - acc: 0.8763\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6801 - acc: 0.8816\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6782 - acc: 0.8816\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6763 - acc: 0.8816\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Adam, total=   1.6s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6928 - acc: 0.5778\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6892 - acc: 0.8127\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6863 - acc: 0.8734\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6836 - acc: 0.8971\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6806 - acc: 0.9103\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6773 - acc: 0.9129\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6736 - acc: 0.9103\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6695 - acc: 0.9182\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6650 - acc: 0.9235\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6600 - acc: 0.9261\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 12us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   1.6s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6955 - acc: 0.2401\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6922 - acc: 0.6544\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6897 - acc: 0.8865\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6872 - acc: 0.9077\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6845 - acc: 0.9156\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6815 - acc: 0.9208\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6780 - acc: 0.9182\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6740 - acc: 0.9235\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6695 - acc: 0.9235\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6646 - acc: 0.9208\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 11us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   1.6s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 2ms/step - loss: 0.6875 - acc: 0.7947\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6831 - acc: 0.8553\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6798 - acc: 0.8658\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6765 - acc: 0.8737\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6730 - acc: 0.8789\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6692 - acc: 0.8868\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6651 - acc: 0.8895\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6605 - acc: 0.8921\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6556 - acc: 0.8921\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6502 - acc: 0.8921\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=uniform, optimizer=Nadam, total=   1.7s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 2ms/step - loss: 0.6873 - acc: 0.7441\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6845 - acc: 0.8179\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6817 - acc: 0.8602\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6788 - acc: 0.8734\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6760 - acc: 0.8918\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6732 - acc: 0.9129\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6703 - acc: 0.9156\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6674 - acc: 0.9288\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6645 - acc: 0.9314\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6615 - acc: 0.9261\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   1.7s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6905 - acc: 0.6174\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6885 - acc: 0.7071\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6866 - acc: 0.7520\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6846 - acc: 0.7968\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6826 - acc: 0.8311\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6805 - acc: 0.8575\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6784 - acc: 0.8707\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6763 - acc: 0.8813\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6741 - acc: 0.8918\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6719 - acc: 0.8971\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   2.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 3ms/step - loss: 0.6870 - acc: 0.7921\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6850 - acc: 0.8184\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6829 - acc: 0.8421\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6809 - acc: 0.8605\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6788 - acc: 0.8737\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6766 - acc: 0.8816\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6744 - acc: 0.8789\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6722 - acc: 0.8842\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6699 - acc: 0.8895\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6675 - acc: 0.8895\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Adam, total=   1.8s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.7012 - acc: 0.1953\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6969 - acc: 0.3245\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6938 - acc: 0.5013\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6908 - acc: 0.6860\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6878 - acc: 0.7968\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6846 - acc: 0.8813\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6812 - acc: 0.9472\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6777 - acc: 0.9604\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6738 - acc: 0.9631\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6696 - acc: 0.9631\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 6us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   1.9s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6933 - acc: 0.4987\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6891 - acc: 0.7546\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6859 - acc: 0.8470\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6830 - acc: 0.8865\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6799 - acc: 0.8971\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6768 - acc: 0.9129\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6733 - acc: 0.9129\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6696 - acc: 0.9208\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6656 - acc: 0.9235\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6612 - acc: 0.9182\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 12us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   1.9s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 3ms/step - loss: 0.6912 - acc: 0.6184\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6872 - acc: 0.8184\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6842 - acc: 0.8658\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6813 - acc: 0.9079\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6781 - acc: 0.9237\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6747 - acc: 0.9368\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6708 - acc: 0.9368\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6666 - acc: 0.9342\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6620 - acc: 0.9316\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6569 - acc: 0.9263\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=8, kernel_initializer=normal, optimizer=Nadam, total=   2.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6899 - acc: 0.6755\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6877 - acc: 0.8259\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6855 - acc: 0.9077\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6833 - acc: 0.9420\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6811 - acc: 0.9604\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6789 - acc: 0.9631\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6766 - acc: 0.9604\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6743 - acc: 0.9551\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6719 - acc: 0.9551\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6695 - acc: 0.9551\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 12us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   2.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6934 - acc: 0.4037\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6917 - acc: 0.6570\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6901 - acc: 0.8496\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6884 - acc: 0.9235\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6867 - acc: 0.9367\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6849 - acc: 0.9578\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6831 - acc: 0.9604\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6812 - acc: 0.9578\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6793 - acc: 0.9499\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6773 - acc: 0.9420\n",
      "190/190 [==============================] - 0s 2ms/step\n",
      "379/379 [==============================] - 0s 12us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   2.0s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 3ms/step - loss: 0.6986 - acc: 0.1474\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6967 - acc: 0.2474\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6948 - acc: 0.4000\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6929 - acc: 0.5395\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6910 - acc: 0.6289\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 5us/step - loss: 0.6891 - acc: 0.7211\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6872 - acc: 0.7947\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6852 - acc: 0.8342\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6832 - acc: 0.8526\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6811 - acc: 0.8711\n",
      "189/189 [==============================] - 0s 2ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Adam, total=   2.1s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6948 - acc: 0.3298\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6908 - acc: 0.7784\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6879 - acc: 0.9288\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6851 - acc: 0.9578\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6821 - acc: 0.9736\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6789 - acc: 0.9657\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6753 - acc: 0.9657\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6712 - acc: 0.9657\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6667 - acc: 0.9631\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6616 - acc: 0.9604\n",
      "190/190 [==============================] - 0s 3ms/step\n",
      "379/379 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   2.2s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6978 - acc: 0.1372\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6944 - acc: 0.3219\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6919 - acc: 0.6491\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6894 - acc: 0.8259\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6868 - acc: 0.8786\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6838 - acc: 0.9024\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6805 - acc: 0.9129\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6769 - acc: 0.9182\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 10us/step - loss: 0.6729 - acc: 0.9182\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6683 - acc: 0.9182\n",
      "190/190 [==============================] - 0s 3ms/step\n",
      "379/379 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   2.5s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 3ms/step - loss: 0.6935 - acc: 0.4579\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6908 - acc: 0.7763\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 6us/step - loss: 0.6887 - acc: 0.8474\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6865 - acc: 0.8816\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6841 - acc: 0.9026\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6813 - acc: 0.9132\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6781 - acc: 0.9184\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6744 - acc: 0.9184\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6701 - acc: 0.9211\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6653 - acc: 0.9211\n",
      "189/189 [==============================] - 1s 3ms/step\n",
      "380/380 [==============================] - 0s 4us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=uniform, optimizer=Nadam, total=   2.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 3ms/step - loss: 0.6778 - acc: 0.9103\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6752 - acc: 0.9182\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6726 - acc: 0.9288\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6698 - acc: 0.9314\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6671 - acc: 0.9288\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6643 - acc: 0.9288\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 13us/step - loss: 0.6615 - acc: 0.9288\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6585 - acc: 0.9288\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6556 - acc: 0.9314\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6525 - acc: 0.9288\n",
      "190/190 [==============================] - 1s 3ms/step\n",
      "379/379 [==============================] - 0s 12us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   2.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 4ms/step - loss: 0.6796 - acc: 0.7546\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6768 - acc: 0.7968\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6740 - acc: 0.8259\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 5us/step - loss: 0.6712 - acc: 0.8470\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6684 - acc: 0.8628\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6656 - acc: 0.8813\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6628 - acc: 0.8839\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6600 - acc: 0.8865\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6572 - acc: 0.8918\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6544 - acc: 0.8945\n",
      "190/190 [==============================] - 1s 3ms/step\n",
      "379/379 [==============================] - 0s 13us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   2.3s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 4ms/step - loss: 0.6943 - acc: 0.4132\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 11us/step - loss: 0.6912 - acc: 0.5132\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6881 - acc: 0.6316\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6851 - acc: 0.7263\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6820 - acc: 0.7868\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6789 - acc: 0.8289\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6757 - acc: 0.8579\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6726 - acc: 0.8789\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6694 - acc: 0.8895\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 7us/step - loss: 0.6661 - acc: 0.9000\n",
      "189/189 [==============================] - 1s 3ms/step\n",
      "380/380 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Adam, total=   2.4s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 1s 4ms/step - loss: 0.6934 - acc: 0.5330\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 12us/step - loss: 0.6872 - acc: 0.6807\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6825 - acc: 0.7203\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6780 - acc: 0.7652\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6735 - acc: 0.7889\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6686 - acc: 0.8259\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 6us/step - loss: 0.6635 - acc: 0.8602\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6581 - acc: 0.8786\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6523 - acc: 0.8865\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6461 - acc: 0.8945\n",
      "190/190 [==============================] - 1s 3ms/step\n",
      "379/379 [==============================] - 0s 8us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   2.5s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "379/379 [==============================] - 2s 4ms/step - loss: 0.6937 - acc: 0.5092\n",
      "Epoch 2/10\n",
      "379/379 [==============================] - 0s 11us/step - loss: 0.6872 - acc: 0.6834\n",
      "Epoch 3/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6823 - acc: 0.7704\n",
      "Epoch 4/10\n",
      "379/379 [==============================] - 0s 7us/step - loss: 0.6776 - acc: 0.8127\n",
      "Epoch 5/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6727 - acc: 0.8522\n",
      "Epoch 6/10\n",
      "379/379 [==============================] - 0s 8us/step - loss: 0.6676 - acc: 0.8813\n",
      "Epoch 7/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6622 - acc: 0.8865\n",
      "Epoch 8/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6565 - acc: 0.9050\n",
      "Epoch 9/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6505 - acc: 0.9129\n",
      "Epoch 10/10\n",
      "379/379 [==============================] - 0s 9us/step - loss: 0.6442 - acc: 0.9182\n",
      "190/190 [==============================] - 1s 3ms/step\n",
      "379/379 [==============================] - 0s 5us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   2.5s\n",
      "[CV] activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam \n",
      "Epoch 1/10\n",
      "380/380 [==============================] - 1s 4ms/step - loss: 0.6903 - acc: 0.5868\n",
      "Epoch 2/10\n",
      "380/380 [==============================] - 0s 12us/step - loss: 0.6828 - acc: 0.7158\n",
      "Epoch 3/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6772 - acc: 0.7842\n",
      "Epoch 4/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6718 - acc: 0.8263\n",
      "Epoch 5/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6663 - acc: 0.8658\n",
      "Epoch 6/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6606 - acc: 0.8763\n",
      "Epoch 7/10\n",
      "380/380 [==============================] - 0s 10us/step - loss: 0.6547 - acc: 0.8842\n",
      "Epoch 8/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6483 - acc: 0.8921\n",
      "Epoch 9/10\n",
      "380/380 [==============================] - 0s 9us/step - loss: 0.6417 - acc: 0.8921\n",
      "Epoch 10/10\n",
      "380/380 [==============================] - 0s 8us/step - loss: 0.6346 - acc: 0.9000\n",
      "189/189 [==============================] - 1s 3ms/step\n",
      "380/380 [==============================] - 0s 13us/step\n",
      "[CV]  activation=elu, batch_size=1024, dropout_rate=0.0, epochs=10, first_neuron=9, kernel_initializer=normal, optimizer=Nadam, total=   2.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:  1.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "569/569 [==============================] - 2s 3ms/step - loss: 0.6932 - acc: 0.4780\n",
      "Epoch 2/10\n",
      "569/569 [==============================] - 0s 10us/step - loss: 0.6910 - acc: 0.6116\n",
      "Epoch 3/10\n",
      "569/569 [==============================] - 0s 7us/step - loss: 0.6888 - acc: 0.7241\n",
      "Epoch 4/10\n",
      "569/569 [==============================] - 0s 5us/step - loss: 0.6866 - acc: 0.7979\n",
      "Epoch 5/10\n",
      "569/569 [==============================] - 0s 5us/step - loss: 0.6843 - acc: 0.8348\n",
      "Epoch 6/10\n",
      "569/569 [==============================] - 0s 6us/step - loss: 0.6820 - acc: 0.8594\n",
      "Epoch 7/10\n",
      "569/569 [==============================] - 0s 6us/step - loss: 0.6797 - acc: 0.8805\n",
      "Epoch 8/10\n",
      "569/569 [==============================] - 0s 6us/step - loss: 0.6773 - acc: 0.8893\n",
      "Epoch 9/10\n",
      "569/569 [==============================] - 0s 9us/step - loss: 0.6749 - acc: 0.8998\n",
      "Epoch 10/10\n",
      "569/569 [==============================] - 0s 5us/step - loss: 0.6724 - acc: 0.9086\n"
     ]
    }
   ],
   "source": [
    "# Perform the Search!\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=1, cv=3, verbose=2)\n",
    "grid_result = grid.fit(x, y) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "\n",
    "Let's see which configuration give us the best performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.938489 using {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'Adam'}\n",
      "0.859402 (0.094322) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'uniform', 'optimizer': 'Adam'}\n",
      "0.896309 (0.043754) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'uniform', 'optimizer': 'Nadam'}\n",
      "0.866432 (0.058552) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'normal', 'optimizer': 'Adam'}\n",
      "0.926186 (0.034017) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'normal', 'optimizer': 'Nadam'}\n",
      "0.887522 (0.055293) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'Adam'}\n",
      "0.885765 (0.065705) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'Nadam'}\n",
      "0.901582 (0.019412) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'normal', 'optimizer': 'Adam'}\n",
      "0.885765 (0.030188) with: {'activation': 'relu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'normal', 'optimizer': 'Nadam'}\n",
      "0.875220 (0.039725) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'uniform', 'optimizer': 'Adam'}\n",
      "0.915641 (0.037395) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'uniform', 'optimizer': 'Nadam'}\n",
      "0.917399 (0.021626) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'normal', 'optimizer': 'Adam'}\n",
      "0.917399 (0.020268) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 8, 'kernel_initializer': 'normal', 'optimizer': 'Nadam'}\n",
      "0.938489 (0.013885) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'Adam'}\n",
      "0.929701 (0.025109) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'uniform', 'optimizer': 'Nadam'}\n",
      "0.913884 (0.027976) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'normal', 'optimizer': 'Adam'}\n",
      "0.906854 (0.049824) with: {'activation': 'elu', 'batch_size': 1024, 'dropout_rate': 0.0, 'epochs': 10, 'first_neuron': 9, 'kernel_initializer': 'normal', 'optimizer': 'Nadam'}\n"
     ]
    }
   ],
   "source": [
    "# Show results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**That's all folks - don't forget to shutdown your workspace once you're done 🙂**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}