IFT6266 Kaggle.ipynb

{
 "metadata": {
  "name": "IFT6266 Kaggle"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Emotion and identity detection from face images"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import contest_dataset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Using gpu device 0: GeForce GTX 460\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t = contest_dataset.ContestDataset('train')\n",
      "t.num_examples"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "4178"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataset = \"\"\"!obj:contest_dataset.ContestDataset {\n",
      "        which_set: 'train',\n",
      "        start: 0,\n",
      "        stop: 3000\n",
      "    }\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = \"\"\"!obj:pylearn2.models.mlp.MLP {\n",
      "        layers: [!obj:pylearn2.models.mlp.Softmax {\n",
      "                     layer_name: 'y',\n",
      "                     n_classes: 7,\n",
      "                     irange: 0.\n",
      "                 }\n",
      "                ],\n",
      "        nvis: 2304,\n",
      "    }\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "algorithm = \"\"\"!obj:pylearn2.training_algorithms.bgd.BGD {\n",
      "        batch_size: 100,\n",
      "        line_search_mode: 'exhaustive',\n",
      "        conjugate: 1,\n",
      "        monitoring_dataset:\n",
      "            {\n",
      "                'train' : *train,\n",
      "                'valid' : !obj:contest_dataset.ContestDataset {\n",
      "                              which_set: 'train',\n",
      "                              start: 3001,\n",
      "                              stop: 3500\n",
      "                          },\n",
      "                'test'  : !obj:contest_dataset.ContestDataset {\n",
      "                              which_set: 'train',\n",
      "                              start: 3501,\n",
      "                              stop: 4170\n",
      "                          }\n",
      "            },\n",
      "        cost: !obj:pylearn2.costs.cost.MethodCost {\n",
      "                method: 'cost_from_X',\n",
      "                supervised: 1\n",
      "        },\n",
      "        termination_criterion: !obj:pylearn2.termination_criteria.MonitorBased {\n",
      "            channel_name: \"valid_y_misclass\"\n",
      "        }\n",
      "    }\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train = \"\"\"!obj:pylearn2.train.Train {\n",
      "    dataset: &train %(dataset)s,\n",
      "    model: %(model)s,\n",
      "    algorithm: %(algorithm)s,\n",
      "    extensions: [\n",
      "        !obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest {\n",
      "             channel_name: 'valid_y_misclass',\n",
      "             save_path: \"softmax_regression_best.pkl\"\n",
      "        },\n",
      "    ],\n",
      "    save_path: \"softmax_regression.pkl\",\n",
      "    save_freq: 1\n",
      "}\"\"\" % locals()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pylearn2.config import yaml_parse\n",
      "train = yaml_parse.load(train)\n",
      "train.main_loop()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "/home/xavier/data/ContestDataset/train.csv\n",
        "Emotion,Pixels\r\n",
        "\n",
        "/home/xavier/data/ContestDataset/train.csv"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Emotion,Pixels\r\n",
        "\n",
        "/home/xavier/data/ContestDataset/train.csv"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Emotion,Pixels\r\n",
        "\n",
        "compiling begin_record_entry...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "compiling begin_record_entry done. Time elapsed: 0.122889 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitored channels: \n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling accum...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 77\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 74\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 74\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling accum done. Time elapsed: 3.052463 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 1.9459104538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.142857283354\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.877428889275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 1.9459104538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.94591033459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.142857298255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.897666752338\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.94591033459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 1.94591021538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.142857298255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.903807640076\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.94591021538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.714619 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 1\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 30\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 3000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 5.91456284837e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 465.503387451\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000203225761652\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.994712352753\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.00926951877773\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.00688401469961\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.00520754233003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.979307770729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.558581471443\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.229062572122\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.327354252338\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.994712352753\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00120034266729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000346760178218\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 6.01429783273e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.953267991543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.00926951877773\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.00688401516527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.00520754186437\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.992903113365\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.560972332954\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.227744743228\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.306333363056\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.953267991543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00120034301654\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000346760207321\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 6.01429856033e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.9680300951\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.00926951970905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.00688401516527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.00520754326135\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.993461668491\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.557750165462\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.227676346898\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.332665324211\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.9680300951\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00120034266729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000346760178218\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 6.01429746894e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.012223 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.638816 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 2\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 60\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 6000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 6.38787128082e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 389.714477539\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000200747686904\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.922323167324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0122104287148\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.00998417939991\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.00806702114642\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.994575500488\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.622778534889\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.249108687043\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.300448417664\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.922323167324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00204435689375\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000498675100971\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 7.53621134209e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.831826508045\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0122104287148\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.00998417939991\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.00806702207774\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.994360685349\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.633506655693\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.243413239717\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.273333281279\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.831826508045\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00204435689375\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000498675159179\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 7.53621134209e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.907090723515\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0122104287148\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.00998417939991\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.00806702114642\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.996843457222\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.621808886528\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.231710925698\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.294589191675\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.907090723515\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00204435689375\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000498675159179\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 7.53621206968e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.010356 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.629213 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 3\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 90\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 9000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 7.0724604484e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 359.478820801\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.00020311769913\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.832639336586\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0141822351143\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0121205346659\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0101538617164\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.994897544384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.658758819103\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.256663113832\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.291479825974\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.832639336586\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00255985674448\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000606863235589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 6.5373875259e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.736401855946\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0141822351143\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0121205374599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.010153863579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.99803185463\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.665902137756\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.253101557493\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.233666688204\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.736401855946\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00255985674448\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000606863235589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 6.53738534311e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.850854158401\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0141822351143\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0121205346659\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0101538617164\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999237418175\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.655753433704\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.254004299641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.306613206863\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.850854158401\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00255985697731\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000606863235589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 6.5373875259e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.013253 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.681711 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 4\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 120\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 12000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 8.0260372215e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 348.474273682\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.00021634506993\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.833844721317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0160628557205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0142324268818\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0124370846897\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.997181117535\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.687212646008\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.262766450644\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.294469386339\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.833844721317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.003112194594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000712117995135\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 9.14183474379e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.685063660145\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0160628557205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0142324231565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0124370874837\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.998688578606\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.691180586815\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.264783143997\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.224333360791\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.685063660145\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00311219412833\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000712117820513\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 9.14183474379e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.817737162113\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0160628557205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0142324268818\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0124370846897\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.99939095974\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.679798662663\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.275676578283\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.278557121754\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.817737162113\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.003112194594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000712118053343\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 9.14183474379e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.013574 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.673984 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 150\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 15000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 7.5713012393e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 321.562042236\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.00019056131714\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.851621508598\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0174945313483\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0157682597637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0138821853325\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.998538911343\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.676082909107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.246783748269\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.284005999565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.851621508598\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00347245950252\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000789903861005\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.000155991583597\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.672613620758\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0174945350736\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0157682579011\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0138821862638\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999067425728\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.686919748783\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.262488394976\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.209333330393\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.672613620758\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00347246020101\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000789903628174\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.000155991583597\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.8194231987\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0174945313483\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0157682597637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0138821853325\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999439120293\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.675471663475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.240896314383\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.264529079199\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.8194231987\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00347245973535\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000789903861005\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.000155991583597\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.013819 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.551007 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 6\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 180\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 18000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 7.70566828123e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 300.354217529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000184995034942\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.807301104069\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0188948251307\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0173030029982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0154668372124\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.99854528904\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.703854262829\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.281694740057\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.267563521862\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.807301104069\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.0038683200255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000866745423991\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.000187954283319\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.640102267265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0188948232681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0173030067235\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0154668409377\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999351620674\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.715818166733\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.269088149071\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.206000030041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.640102267265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00386831955984\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000866745132953\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.000187954254216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.818853259087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0188948232681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0173030029982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0154668372124\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999635517597\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.69771361351\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.276006281376\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.280561119318\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.818853259087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.0038683200255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000866745482199\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.000187954268767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.014074 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.499694 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 7\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 210\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 21000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 8.37586640046e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 285.496185303\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000192950756173\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.806085884571\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0201019514352\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0186963044107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0169290788472\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.997776925564\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.727297842503\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.268051832914\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.27503734827\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.806085884571\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00416887039319\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.000937960110605\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.000183043754078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.599446117878\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0201019477099\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0186963044107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0169290769845\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999654769897\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.734404444695\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.288558274508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.194333314896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.599446117878\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00416887085885\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.000937959819566\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.000183043739526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.824137568474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0201019495726\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0186963044107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0169290788472\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999806642532\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.719401836395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.254995375872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.280561089516\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.824137568474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00416887039319\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.000937960110605\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.000183043754078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.014675 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.394529 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 8\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 240\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 24000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 9.26486450226e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 300.042877197\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000217626715312\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.821137428284\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0215442962945\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0202026814222\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0184498429298\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.99858880043\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.740391612053\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.281957298517\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.276532173157\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.821137428284\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00456400914118\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.00101419701241\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0002160363656\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.594242274761\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0215442907065\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0202026795596\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0184498485178\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999608874321\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.753973782063\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.294180065393\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.201000019908\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.594242274761\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00456400820985\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.00101419677958\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.000216036452912\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.851036131382\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0215442962945\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0202026814222\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0184498429298\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999807536602\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.741039633751\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.279352009296\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.274549096823\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.851036131382\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00456400914118\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.00101419701241\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.000216036351048\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.014648 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 8.645156 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 9\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 270\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 27000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 9.10431367629e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 290.612701416\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000209268750041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.844288766384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0229641851038\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0214670524001\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0198403056711\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999464273453\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.737448573112\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.310814142227\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.281016469002\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.844288766384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00484321219847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.00107755418867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.000219538822421\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.573635518551\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0229641813785\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0214670561254\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0198403019458\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999868631363\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.749578714371\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.282893270254\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.189999982715\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.573635518551\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00484321219847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.00107755430508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.000219538749661\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.824259757996\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0229641832411\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0214670505375\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0198403056711\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.99994301796\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.729735851288\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.287635415792\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.276553124189\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.824259757996\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00484321219847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.00107755418867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.000219538851525\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.012758 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 9.136012 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 300\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 30000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_mult: 9.56724420575e-07\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_grad_size: 276.747558594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tave_step_size: 0.000206592914765\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 0.83696013689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0243078563362\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.022726200521\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0210664458573\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999303102493\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.718197464943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.291966319084\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.294469356537\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 0.83696013689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.00509882858023\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.00114204222336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.000214216095628\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 0.565506100655\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0243078544736\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0227261949331\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0210664439946\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999821484089\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.737155377865\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.269201219082\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.183999970555\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.565506100655\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.00509882764891\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.00114204164129\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.00021421611018\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 0.817026138306\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0243078563362\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.022726200521\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0210664458573\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999872386456\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.711395561695\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.27066463232\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.282565146685\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 0.817026138306\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.00509882904589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.00114204222336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.00021421611018\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.015594 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to softmax_regression.pkl done. Time elapsed: 0.015826 seconds\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = \"\"\"!obj:pylearn2.models.mlp.MLP {\n",
      "        layers: [ !obj:pylearn2.models.mlp.RectifiedLinear {\n",
      "                     layer_name: 'h0',\n",
      "                     dim: 2000,\n",
      "                     sparse_init: 15,\n",
      "                 },  !obj:pylearn2.models.mlp.RectifiedLinear {\n",
      "                     layer_name: 'h1',\n",
      "                     dim: 2304,\n",
      "                     sparse_init: 15,\n",
      "                 }, !obj:pylearn2.models.mlp.Softmax {\n",
      "                     layer_name: 'y',\n",
      "                     n_classes: 7,\n",
      "                     irange: 0.\n",
      "                 }\n",
      "                ],\n",
      "        nvis: 2304,\n",
      "    }\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "algorithm = \"\"\"!obj:pylearn2.training_algorithms.sgd.SGD {\n",
      "        batch_size: 100,\n",
      "        learning_rate: .0001,\n",
      "        init_momentum: .5,\n",
      "\n",
      "        monitoring_dataset:\n",
      "            {\n",
      "                'train' : *train,\n",
      "                'valid' : !obj:contest_dataset.ContestDataset {\n",
      "                              which_set: 'train',\n",
      "                              start: 3001,\n",
      "                              stop: 3500\n",
      "                          },\n",
      "                'test'  : !obj:contest_dataset.ContestDataset {\n",
      "                              which_set: 'train',\n",
      "                              start: 3501,\n",
      "                              stop: 4170\n",
      "                          }\n",
      "            },\n",
      "\n",
      "\n",
      "        cost: !obj:pylearn2.costs.cost.SumOfCosts { costs: [\n",
      "            !obj:pylearn2.costs.cost.MethodCost {\n",
      "                method: 'cost_from_X',\n",
      "                supervised: 1\n",
      "            }, !obj:pylearn2.models.mlp.WeightDecay {\n",
      "                coeffs: [ .000005, .00000005, .0000000005 ]\n",
      "            }\n",
      "            ]\n",
      "        },\n",
      "        termination_criterion: !obj:pylearn2.termination_criteria.MonitorBased {\n",
      "            channel_name: \"valid_y_misclass\",\n",
      "            prop_decrease: 0.,\n",
      "            N: 20\n",
      "        }\n",
      "    }\"\"\"\n",
      "\n",
      "\n",
      "\n",
      "#algorithm = \"\"\"!obj:pylearn2.training_algorithms.sgd.SGD {\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "#    }\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 73
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train = \"\"\"!obj:pylearn2.train.Train {\n",
      "    dataset: &train %(dataset)s,\n",
      "    model: %(model)s,\n",
      "    algorithm: %(algorithm)s,\n",
      "    extensions: [\n",
      "        !obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest {\n",
      "             channel_name: 'valid_y_misclass',\n",
      "             save_path: \"mlp_3_best.pkl\"\n",
      "        },!obj:pylearn2.training_algorithms.sgd.MomentumAdjustor {\n",
      "            start: 1,\n",
      "            saturate: 10,\n",
      "            final_momentum: .99\n",
      "        }\n",
      "    ],\n",
      "    save_path: \"mlp_3_regression.pkl\",\n",
      "    save_freq: 1\n",
      "}\"\"\" % locals()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 74
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pylearn2.config import yaml_parse\n",
      "train = yaml_parse.load(train)\n",
      "train.main_loop()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Parameter and initial learning rate summary:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\th0_W: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\th0_b: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\th1_W: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\th1_b: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tsoftmax_b: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tsoftmax_W: 9.99999974738e-05\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling sgd_update...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling sgd_update done. Time elapsed: 1.691802 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "compiling begin_record_entry...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "compiling begin_record_entry done. Time elapsed: 0.261423 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitored channels: \n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling accum...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 171\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 169\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "graph size: 169\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Compiling accum done. Time elapsed: 7.897916 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07625627518\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78453207016\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54955112934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53333902359\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48074102402\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.680733025074\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33723306656\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79568910599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48913657665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87341451645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488967896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40597820282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.09572768211\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 1.9459104538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149817287922\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.142857283354\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.877428889275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 1.9459104538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07625579834\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.784532547\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54955136776\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53333616257\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48074173927\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.680733263493\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33723402023\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79569005966\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48913705349\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87341117859\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489063263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40597820282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.09572768211\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.94591033459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149817287922\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.142857298255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.897666633129\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.94591033459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07625675201\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.784532547\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54955112934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53333950043\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48074150085\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.680733025074\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33723306656\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79568910599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48913657665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87341451645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40597832203\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.0957274437\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.94591021538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149817287922\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.142857298255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.142857149243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.903807640076\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.94591021538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.890009 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 1\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 30\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 3000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07619190216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78554296494\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491232872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53863477707\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.4819817543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691078841686\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555940628\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891162872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05489063263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.4058817625\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 5.68165588379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 5.53176593781\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149890065193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82611179352\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89713716507\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.932623505592\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 1.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.653132855892\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.22017967701\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.565022408962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 5.53176593781\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0799390524626\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07619333267\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554153442\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912316799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53863573074\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48198103905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691078782082\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79556035995\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891198635\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.0548915863\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.4058817625\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 4.1205997467\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 3.97070980072\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149890065193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82611083984\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89713716507\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.932623684406\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 1.00000011921\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.664435505867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.232297420502\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.518666684628\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 3.97070980072\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0799390152097\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07619190216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554296494\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.5491232872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53863430023\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48198151588\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691078841686\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555940628\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891162872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05489110947\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.4058817625\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 3.49955940247\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 3.34966945648\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149890065193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.8261115551\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89713716507\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.932623565197\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999999940395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.657190263271\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.207606688142\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.569138288498\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 3.34966945648\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.079939045012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.515317 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.859061 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 2\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 60\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 6000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.554444432259\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07615375519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78554153442\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912292957\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53863525391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48198032379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.69110918045\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555797577\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891294003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585255623\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 4.1924238205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 4.04253387451\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.14988990128\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82473611832\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89580738544\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.929570913315\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999999165535\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.669580936432\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.254196792841\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.499252617359\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 4.04253387451\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0799287185073\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07615423203\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554129601\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912257195\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53863573074\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197960854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.69110918045\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891234398\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585279465\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.30608725548\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 2.15619707108\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.14988990128\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82473683357\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89580690861\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.929571032524\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.99999153614\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.681014835835\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.264586418867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.453999996185\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 2.15619707108\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0799287110567\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07615375519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554153442\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912292957\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53863525391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48198056221\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.69110918045\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555773735\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891270161\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488967896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585243702\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.6833896637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.53349995613\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.14988990128\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82473659515\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89580726624\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.929570972919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999995589256\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.67942404747\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.24630741775\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.533066153526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.53349995613\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0799287185073\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.661378 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.878545 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 3\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 90\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 9000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.608888864517\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07612848282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78554105759\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912090302\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53864431381\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197984695\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691124975681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555654526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891425133\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585386753\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.471347332\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 3.32145762444\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889782071\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82423758507\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89523100853\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.928963005543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999998867512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.687503218651\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.250682622194\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.48430493474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 3.32145762444\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0799490958452\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07612705231\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554081917\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912126064\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53864336014\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691124856472\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891460896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585339069\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.63184547424\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.48195564747\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889841676\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82423686981\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89523112774\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.928963303566\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999997496605\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.70000910759\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.245028346777\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.40433332324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.48195564747\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0799490734935\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07612800598\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554105759\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912090302\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53864431381\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691124975681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555702209\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891437054\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585398674\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.32843160629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.17854189873\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889796972\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82423734665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89523112774\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.928963005543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999998927116\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.690007328987\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.240529641509\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.496994018555\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.17854189873\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0799490958452\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.636374 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.873207 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 4\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 120\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 12000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.663333296776\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07612419128\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553986549\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491206646\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.5386505127\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.4819791317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691134393215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585565567\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.13393163681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.98404192924\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889737368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82390117645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89491903782\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.928723692894\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999996244907\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.660769581795\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.23288667202\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.443946182728\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.98404192924\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0799944773316\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07612514496\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554081917\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912126064\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53865242004\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691134333611\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585625172\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.29635465145\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.14646530151\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889737368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.8239004612\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89491963387\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.928723752499\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999978780746\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.675800979137\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.240424275398\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.365000039339\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.14646530151\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0799944922328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07612371445\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554034233\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.5491206646\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53865098953\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.69113445282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585577488\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.19562506676\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.04573535919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889737368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82390117645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89491915703\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.928723692894\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999996483326\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.662300229073\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.223277777433\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.494989991188\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.04573535919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0799944847822\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.666234 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.946558 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 150\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 15000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.717777788639\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07612323761\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553986549\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912054539\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53865432739\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197889328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691139578819\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585589409\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.98300600052\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.83311629295\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889677763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.8237118721\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89482748508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.928278923035\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999998986721\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.681810200214\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.251228094101\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.434977561235\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.83311629295\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0800472646952\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07612514496\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554058075\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911983013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53865385056\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.4819791317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691139578819\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05488681793\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585637093\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.08818805218\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 0.938298344612\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889707565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82371282578\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89482700825\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.928278684616\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999988734722\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.693039119244\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.265051245689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.309000045061\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.938298344612\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0800472572446\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07612371445\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554010391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912042618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53865432739\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.4819791317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691139638424\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891568184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.4058560133\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.08877754211\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.93888783455\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889677763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.8237118721\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89482748508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.928278923035\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999997138977\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.685521364212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.249143242836\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.456913858652\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.93888783455\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0800472646952\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.856981 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.886272 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 6\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 180\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 18000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.772222220898\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07611608505\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553962708\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912030697\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53865528107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197865486\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691132009029\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585649014\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.09467816353\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.94478821754\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.14988963306\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82358312607\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89480614662\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.928400456905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999994516373\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.75517308712\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.27738109231\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.448430478573\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.94478821754\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0801164358854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07611560822\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554058075\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911983013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53865385056\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.4819791317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691132009029\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891556263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05488681793\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585649014\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.18600642681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.03611671925\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889618158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82358407974\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89480566978\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.928400278091\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 1.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.762295842171\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.284581959248\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.334999978542\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.03611671925\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.080116443336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07611608505\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78553986549\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912030697\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53865528107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197889328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691132068634\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585649014\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.32312774658\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.17323803902\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.14988963306\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82358312607\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89480614662\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.92840051651\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999999940395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.746765613556\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.263233274221\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.470941871405\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.17323803902\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0801164209843\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.590085 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.917174 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 7\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 210\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 21000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.826666653156\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07613658905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553843498\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491194725\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53865861893\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197865486\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691134393215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488920212\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585660934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.06276011467\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.9128704071\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889588356\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82349205017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89491772652\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.928801834583\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999999403954\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.674649298191\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.257403731346\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.484304904938\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.9128704071\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0801796540618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07613945007\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78553962708\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911911488\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53865671158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.4819791317\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691134333611\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891556263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585660934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.06930994987\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 0.919420480728\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889588356\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82349228859\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89491713047\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.928802073002\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999998271465\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.694798052311\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.247689977288\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.333666712046\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 0.919420480728\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0801796540618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07613706589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.7855386734\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911959171\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53865861893\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197889328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.69113445282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05488967896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585660934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.14548873901\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.99559891224\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889588356\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82349205017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89491796494\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.928801953793\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999999940395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.682295799255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.254655689001\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.466933876276\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.99559891224\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0801796540618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.651764 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.877064 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 8\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 240\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 24000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.88111114502\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07613611221\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553819656\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911935329\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53866863251\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197770119\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691131472588\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555654526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05488967896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585660934\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.02706718445\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.87717747688\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889498949\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82347559929\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89514124393\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.929336249828\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999999284744\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.662613868713\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.246889799833\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.515695095062\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.87717747688\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.080285154283\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07613706589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78553938866\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911899567\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53866815567\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197889328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691131472588\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555511475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891556263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585672855\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.21877348423\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.06888401508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889454246\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82347655296\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89514148235\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.929336667061\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999999523163\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.683384537697\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.250800341368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.372333288193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.06888401508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.080285154283\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07613658905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78553795815\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911935329\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53866863251\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197793961\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691131472588\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555654526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05489015579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585672855\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.47307920456\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.32318949699\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889498949\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82347536087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89514136314\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.929336309433\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999999940395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.662061929703\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.253577739\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.541082143784\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.32318949699\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0802851617336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.624900 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.909778 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 9\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 270\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 27000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.935555577278\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07606124878\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553676605\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911851883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.5386557579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197698593\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691137373447\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79555797577\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05489110947\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585780144\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.92632770538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.77643847466\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889364839\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.82343316078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89554619789\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.930443406105\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999999880791\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.673044979572\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.235593929887\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.500747382641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.77643847466\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0804772749543\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07605886459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78553557396\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911863804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53865432739\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197746277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691137373447\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79555630684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48891556263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.0548915863\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585768223\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.30020546913\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.1503162384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889394641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82343387604\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89554584026\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.930443584919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999998807907\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.685885131359\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.247079834342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.390666663647\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.1503162384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.080477245152\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07606124878\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78553652763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911839962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53865623474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197698593\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691137433052\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79555773735\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48891544342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05489110947\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585780144\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.30181193352\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.15192270279\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889364839\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82343316078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89554619789\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.930443286896\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 1.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.680899262428\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.228687718511\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.501002013683\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.15192270279\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0804772675037\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.603687 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.862160 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 300\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 30000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07600450516\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78553557396\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911744595\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53870868683\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48197555542\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691139698029\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79556584358\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48898684978\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05490255356\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40586924553\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.24195384979\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 3.09206461906\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889230728\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.8242688179\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.89895606041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.935296356678\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999931514263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.685440480709\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.245766356587\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.579970121384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 3.09206461906\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0813089460135\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07600164413\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78553533554\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911780357\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.5387096405\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48197698593\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691139638424\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79556727409\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.4889870882\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05490112305\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40586972237\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.85024392605\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.70035481453\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889186025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82427048683\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.89895665646\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.935296535492\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 1.00000023842\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.701173484325\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.253439188004\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.496333390474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.70035481453\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0813089385629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07600402832\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78553628922\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911744595\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53870820999\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48197579384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691139698029\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79556584358\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48898673058\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.0549030304\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40586924553\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.88749313354\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.73760390282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889215827\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.82426905632\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.89895606041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.935296356678\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 1.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.698811769485\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.257670760155\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.573146283627\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.73760390282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0813089385629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.540210 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.863947 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 11\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 330\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 33000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07596492767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78554582596\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911983013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53870010376\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48198509216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691279888153\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79560732841\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48912119865\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05495929718\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585017204\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 3.03009104729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.8802011013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149889618158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.8286614418\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.91106569767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 0.957120716572\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999996304512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.60321444273\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.212315618992\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.576980531216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.8802011013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0830523148179\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07596492767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78554654121\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911971092\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53869962692\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48198604584\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691279888153\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79560518265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48912072182\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05495834351\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40584981441\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.06866979599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.91878020763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149889618158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.82866239548\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.9110660553\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 0.957120895386\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 1.00000011921\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.612400591373\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.209063395858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.531333386898\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.91878020763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0830523446202\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07596492767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78554534912\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911983013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53870105743\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48198509216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691279888153\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79560709\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48912119865\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05495929718\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585041046\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.58466315269\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.43477344513\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149889618158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.8286614418\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.91106581688\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 0.957120716572\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.999999523163\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.602419137955\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.196631327271\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.577154278755\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.43477344513\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0830523222685\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.586971 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.880554 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 12\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 360\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 36000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07594490051\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78557491302\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912519455\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53882265091\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48201704025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691634953022\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79569506645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48925352097\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05507802963\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.4058419466\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.52271509171\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.37282323837\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149891674519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.83797311783\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.93321418762\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.00013279915\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.99461877346\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.376146197319\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.143940970302\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.63527649641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.37282323837\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0852341502905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07594394684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78557538986\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912567139\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53882408142\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48201680183\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691634953022\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79569602013\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48925304413\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05507802963\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40584218502\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.87776994705\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.72787857056\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149891674519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.83797478676\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.93321454525\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.00013279915\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.999673128128\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.384803950787\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.143963828683\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.597333371639\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.72787857056\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0852341279387\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07594537735\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78557467461\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912519455\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53882217407\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48201704025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691635012627\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79569506645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48925375938\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05507802963\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40584206581\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.13315415382\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.98326253891\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149891674519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.83797335625\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.93321430683\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.00013267994\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.979333460331\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.375460237265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.143963307142\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.655310630798\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.98326253891\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0852341502905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.561501 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.880155 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 13\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 390\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 39000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07588624954\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78562688828\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912984371\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.5386300087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48207235336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.691990375519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79582905769\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48936963081\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05525302887\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40584445\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.56956624985\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.41967082024\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149895489216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.85180807114\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.96369504929\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.0643248558\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.999791562557\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.270720899105\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.144724816084\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.630792200565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.41967082024\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0874882638454\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07588386536\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78562664986\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912960529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53862810135\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48207235336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.691990315914\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.7958278656\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48936975002\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05525493622\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40584480762\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.04757165909\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.89767622948\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149895519018\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.85180902481\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.9636952877\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.06432509422\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.998522222042\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.287790477276\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.144696936011\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.63033336401\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.89767622948\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0874882861972\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07588624954\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78562641144\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912984371\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.5386300087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48207187653\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.691990435123\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79582881927\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48936975002\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05525350571\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40584456921\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.19728398323\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.04738855362\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149895474315\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.85180830956\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.9636952877\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.06432497501\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.94766253233\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.267564564943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.144783630967\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.677354753017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.04738855362\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.087488271296\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.523826 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.873794 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 14\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 420\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 42000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07583522797\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78568840027\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54913175106\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53856992722\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48213934898\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.692320227623\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79600095749\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48946535587\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05547475815\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40585303307\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.36469173431\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.21479153633\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149900406599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.86815428734\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 1.99593043327\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.11384809017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.952013731003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.251750975847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.155237883329\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.844544053078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.21479153633\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0895383432508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07583618164\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78568983078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54913210869\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.5385684967\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48213839531\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.6923199296\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79599952698\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48946547508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.0554766655\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40585291386\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.04495501518\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.89505469799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149900436401\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.86815357208\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 1.99593102932\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.11384785175\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.979073166847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.256848931313\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.155241921544\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.794333279133\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.89505469799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0895383730531\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.0758357048\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78568887711\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54913175106\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53856992722\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48213911057\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.692320227623\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79600071907\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48946535587\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05547523499\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40585303307\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.19133162498\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.04143118858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149900406599\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.86815428734\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 1.99593043327\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.11384809017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.968704819679\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.243897110224\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.155737668276\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.821643292904\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.04143118858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0895383432508\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.519204 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.876824 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 15\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 450\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 45000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07579469681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78575134277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54913210869\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53850889206\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.4822063446\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.692619681358\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.7962038517\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.4895414114\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05573511124\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40586280823\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.33009696007\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.18019175529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149905309081\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.88694763184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.02787852287\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.15905785561\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.997618377209\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.32735016942\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.14754101634\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.64872944355\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.18019175529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0913959443569\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07579612732\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78574991226\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54913270473\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.5385093689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48220658302\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.692619681358\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79620504379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48954153061\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05573558807\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40586340427\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02846050262\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87855517864\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149905279279\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.88694858551\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.02787852287\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.1590577364\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.998171329498\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.336378455162\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.14780215919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.638333320618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87855517864\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0913958996534\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07579469681\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78575086594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.5491322279\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53850841522\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.4822063446\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.692619681358\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79620337486\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48954164982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05573558807\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40586304665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.15311646461\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 2.00321102142\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149905309081\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.88694763184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.02787876129\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.15905797482\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.986777186394\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.318801522255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.148446261883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.667334675789\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 2.00321102142\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0913959369063\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.543063 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.884984 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 16\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 480\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 48000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.0757598877\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78580570221\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54913127422\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53843259811\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48226475716\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.693039119244\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79642319679\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48960089684\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05601882935\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40587246418\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.16515779495\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.01524806023\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149909719825\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.90623283386\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.05747413635\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.19878280163\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.956348955631\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.24624453485\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.151387065649\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.674140512943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.01524806023\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0929966196418\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07575845718\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78580546379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54913055897\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53843450546\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48226523399\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.693039298058\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79642176628\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48960113525\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05602025986\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40587210655\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.96437430382\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.81446456909\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149909704924\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.90623307228\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.0574734211\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.19878292084\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.979988276958\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.251587569714\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.151253804564\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.673000097275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.81446456909\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0929966345429\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07576036453\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78580522537\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54913115501\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53843307495\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48226499557\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.693039178848\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79642343521\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48960077763\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05601930618\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40587234497\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.11711716652\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.96720719337\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149909719825\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.90623235703\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.05747413635\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.19878292084\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.889269351959\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.23815818131\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.151189774275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.711422860622\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.96720719337\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0929966121912\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.629152 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.891690 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 17\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 510\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 51000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07572984695\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.7858505249\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54913008213\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53836965561\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48231363297\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.693822979927\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79663348198\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48964619637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05629444122\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40588068962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.25442481041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.10451126099\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149913400412\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.92404985428\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.08313298225\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.23176181316\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.909462511539\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.189255282283\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.155238360167\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.702541172504\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.10451126099\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0942769944668\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07572937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78584909439\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.5491297245\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.5383682251\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48231506348\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.693823039532\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79663419724\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48964619637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05629444122\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40588068962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02331233025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87339878082\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149913400412\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.9240500927\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.08313250542\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.23176205158\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.908423900604\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.189252465963\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.155082553625\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.688333392143\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87339878082\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.094276972115\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07573032379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78585004807\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54913008213\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53836965561\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48231434822\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.693823039532\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79663348198\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48964619637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05629444122\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40588057041\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.12965583801\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.9797424078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149913415313\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.92404961586\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.08313274384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.23176181316\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.773784220219\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.177835762501\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.155333206058\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.71943885088\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.9797424078\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0942769944668\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.548075 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.875558 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 18\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 540\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 54000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07570409775\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78588557243\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912781715\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53832054138\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48235225677\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.694420516491\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79681968689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48968064785\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05653905869\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40588724613\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.20977449417\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.05985808372\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149916276336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.93958067894\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.10482645035\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.25891041756\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.963812112808\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.213780418038\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.157765939832\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.689088284969\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.05985808372\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.095317132771\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07570648193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78588461876\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.5491284132\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53832101822\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48235177994\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.694420397282\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79682040215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48968088627\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05653762817\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40588760376\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02258038521\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.8726644516\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149916276336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.93957996368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.10482621193\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.25891065598\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.937831521034\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.21558368206\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.158086597919\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.687333345413\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.8726644516\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0953171551228\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07570457458\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78588581085\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912793636\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53832101822\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48235249519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.6944206357\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79681992531\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48968064785\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05653953552\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40588724613\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.10442233086\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.95450627804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149916276336\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.93958044052\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.10482645035\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.25891053677\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.846156597137\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.200635895133\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.156818851829\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.707414805889\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.95450627804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0953171402216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.543824 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.889779 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 19\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 570\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 57000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07567834854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78591156006\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54912567139\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53827095032\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.482380867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.694868087769\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79697704315\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48970651627\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05674600601\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40589261055\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.18545031548\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.03553199768\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149918407202\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.95258927345\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.12261080742\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.2805839777\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.887200415134\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.186191618443\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.157649248838\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.692077755928\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.03553199768\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0961303710938\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07567882538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78591275215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.5491257906\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53827047348\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48238039017\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.694867908955\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79697751999\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48970627785\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05674505234\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40589249134\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02463960648\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87472116947\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149918451905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.95258903503\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.122610569\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.28058350086\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.883795917034\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.184864044189\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.156766325235\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.685333430767\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87472116947\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0961303636432\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07567834854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.7859120369\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912567139\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53827095032\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.482380867\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.694868028164\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79697680473\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48970663548\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05674600601\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40589261055\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.08218646049\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.93226802349\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149918422103\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.95258927345\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.12261104584\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.2805839777\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.546404242516\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.175612598658\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.153732553124\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.69138276577\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.93226802349\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0961303636432\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.734247 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.893549 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 20\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 600\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 60000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07565689087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78593063354\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491232872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53823661804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48240184784\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.695205688477\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79710388184\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48972606659\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05691432953\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40589666367\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.17870378494\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.02878403664\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149919956923\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.96333146095\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.13699007034\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.29760992527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.930380523205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.191025152802\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.160400435328\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.690582990646\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.02878403664\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0967598110437\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07565498352\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78593254089\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912316799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53823757172\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48240160942\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.695205509663\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79710507393\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48972642422\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05691480637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40589630604\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02731895447\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87739920616\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149919986725\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.96333169937\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.13699007034\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.29760992527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.900725007057\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.190658435225\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.16161775589\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.688666582108\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87739920616\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0967598035932\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07565689087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78593087196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.5491232872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53823661804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48240232468\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.695205688477\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79710412025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.4897261858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05691432953\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40589666367\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.08722186089\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.93730187416\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149919956923\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.96333146095\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.13698983192\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.29760992527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.699392974377\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.185750201344\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.168076917529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.705410838127\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.93730187416\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0967598184943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.710133 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.892360 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 21\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 630\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 63000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07563591003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78594326973\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491206646\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53820562363\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48241615295\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.69546186924\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79720544815\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48974084854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05704879761\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.4058996439\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.12535572052\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 1.97543478012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149921014905\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.97182536125\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.14833569527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.31056320667\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.894633114338\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.204990699887\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.170507580042\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.67563521862\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 1.97543478012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.097231477499\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07563591003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78594207764\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54912126064\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.5382065773\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48241591454\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.695461630821\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79720640182\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48974084854\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05704689026\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.4058996439\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.0064868927\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.8565659523\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149921000004\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.97182393074\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.14833521843\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.31056320667\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.940437793732\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.205832883716\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.172842845321\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.677333295345\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.8565659523\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0972314402461\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07563591003\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78594350815\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54912078381\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53820562363\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48241662979\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.695461928844\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79720520973\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48974072933\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05704879761\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40589976311\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.06452417374\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.91460299492\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149921000004\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.97182559967\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.14833545685\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.31056320667\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.93557959795\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.204233035445\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.173136502504\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.685370743275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.91460299492\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0972314700484\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.533465 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.873297 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 22\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 660\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 66000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07561349869\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78595256805\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911851883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53820705414\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48242688179\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.695656776428\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79728841782\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48975157738\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05715751648\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40590214729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.13738179207\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 1.98745989799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149921745062\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.97849035263\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.15716195107\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.3202881813\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.86745172739\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.200256943703\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.170513346791\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.677130103111\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 1.98745989799\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0975988581777\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07561206818\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78595042229\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911863804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53820848465\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48242807388\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.695657074451\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79728722572\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48975157738\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05715894699\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40590214729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 1.99800848961\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.84808683395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149921670556\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.97848987579\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.15716147423\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.3202880621\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.852408111095\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.200563162565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.167371764779\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.670333385468\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.84808683395\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0975988730788\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07561397552\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78595232964\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911839962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.5382065773\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48242712021\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.695656836033\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.7972881794\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48975157738\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05715847015\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40590214729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.09418511391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.94426345825\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149921730161\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.97849059105\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.15716218948\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.3202881813\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.737613379955\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.194354936481\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.167971149087\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.695390820503\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.94426345825\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0975988507271\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.541039 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.859016 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 23\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 690\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 69000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07560873032\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78595733643\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911470413\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53823280334\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48243355751\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.695806145668\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79735922813\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48975944519\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05725288391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40590405464\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.16722416878\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.01730203629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149922206998\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.98391127586\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.16407895088\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.32761764526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.960133433342\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.206981569529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.172761112452\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.696562111378\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.01730203629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0978945344687\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07561016083\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78595638275\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911494255\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53823423386\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48243522644\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.695806086063\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79735851288\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.4897595644\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05725288391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40590405464\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.0262169838\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87629473209\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149922221899\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.98391056061\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.16407847404\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.32761800289\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.879261791706\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.202991753817\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.168495208025\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.695333302021\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87629473209\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0978945717216\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07560920715\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78595781326\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911470413\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53823232651\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48243379593\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.695806026459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79735922813\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48975932598\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05725288391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40590405464\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.07167434692\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.92175233364\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149922206998\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.9839117527\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.16407871246\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.32761764526\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.85196608305\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.19447222352\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.172676458955\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.713426828384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.92175233364\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0978945344687\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.563822 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.870916 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 24\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 720\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 72000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07568645477\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.7859609127\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.54911136627\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53824710846\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48243808746\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.6959182024\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79747343063\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48976540565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05738115311\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40590536594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.18118095398\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 2.03125834465\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149922505021\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.98804235458\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.16943693161\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.33316886425\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.653886973858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.198738366365\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.188373491168\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.702541172504\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 2.03125834465\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0981321707368\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07568407059\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78596162796\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54911148548\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53824949265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.48243761063\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.695918262005\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79747200012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48976564407\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.05738162994\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40590560436\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.02299928665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.87307715416\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149922505021\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.9880437851\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.16943597794\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.33316886425\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.810284256935\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.199833348393\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.186266809702\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.702666699886\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.87307715416\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.098132185638\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07568645477\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.7859609127\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.54911124706\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.5382475853\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48243832588\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.6959182024\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.7974729538\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48976540565\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05738162994\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40590536594\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.06133770943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.9114151001\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.149922505021\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.988042593\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.16943693161\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.33316886425\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.384474694729\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.193038225174\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.18805103004\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.719438910484\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.9114151001\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0981321632862\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.559124 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Time this epoch: 0.907700 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Monitoring step:\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tEpochs seen: 25\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tBatches seen: 750\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tExamples seen: 75000\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tlearning_rate: 0.00010000000475\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tmomentum: 0.990000069141\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_max: 6.07572984695\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_mean: 3.78596305847\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_col_norms_min: 1.5491065979\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_max: 6.53825569153\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_mean: 3.48244142532\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h0_row_norms_min: 0.696007907391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_mean: 3.79763865471\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_col_norms_min: 1.48977029324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_mean: 4.05755901337\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_h1_row_norms_min: 1.40590643883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_objective: 2.14620375633\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_0: 1.99628090858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_term_1_weight_decay: 0.149922758341\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_max: 3.99137449265\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_mean: 2.17366170883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_col_norms_min: 1.33760213852\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_max_max_class: 0.756652235985\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_mean_max_class: 0.208928108215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_min_max_class: 0.186919853091\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_misclass: 0.702541172504\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_nll: 1.99628090858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_mean: 0.0983239263296\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttest_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_max: 6.07572937012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_mean: 3.78596329689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_col_norms_min: 1.54910695553\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_max: 6.53825378418\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_mean: 3.482442379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h0_row_norms_min: 0.696008086205\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_max: 6.33399438858\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_mean: 3.79763770103\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_col_norms_min: 1.48977029324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_max: 6.87146282196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_mean: 4.0575594902\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_h1_row_norms_min: 1.40590655804\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_objective: 2.01787686348\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_0: 1.86795425415\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_term_1_weight_decay: 0.149922713637\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_max: 3.99137568474\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_mean: 2.17366194725\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_col_norms_min: 1.33760225773\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_max_max_class: 0.81999707222\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_mean_max_class: 0.2101008147\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_min_max_class: 0.180908665061\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_misclass: 0.699000000954\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_nll: 1.86795425415\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_mean: 0.0983239561319\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ttrain_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_max: 6.07573032379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_mean: 3.78596329689\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_col_norms_min: 1.5491065979\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_max: 6.53825521469\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_mean: 3.48244142532\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h0_row_norms_min: 0.696007907391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_max: 6.33399677277\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_mean: 3.79763913155\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_col_norms_min: 1.48977029324\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_max: 6.87146520615\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_mean: 4.05755853653\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_h1_row_norms_min: 1.40590631962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_objective: 2.05796957016\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_0: 1.90804648399\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_term_1_weight_decay: 0.14992274344\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_max: 3.99137473106\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_mean: 2.17366170883\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_col_norms_min: 1.33760213852\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_max_max_class: 0.664192020893\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_mean_max_class: 0.206147298217\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_min_max_class: 0.184421181679\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_misclass: 0.713426828384\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_nll: 1.90804648399\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_max: 0.555937170982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_mean: 0.0983239412308\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\tvalid_y_row_norms_min: 0.0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.570456 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving to mlp_3_regression.pkl done. Time elapsed: 4.628123 seconds\n"
       ]
      }
     ],
     "prompt_number": 75
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    }
   ],
   "metadata": {}
  }
 ]
}