Multilayer fully-connected network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load and split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['pixels', 'overfeat', 'labels', 'names', 'allow_pickle'])\n"
     ]
    }
   ],
   "source": [
    "with np.load('cifar4-train.npz',allow_pickle=False) as npz_file:\n",
    "    data = dict(npz_file.items())\n",
    "    print (data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: (3200, 4096) (3200,)\n",
      "Valid: (800, 4096) (800,)\n",
      "Test: (1000, 4096) (1000,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "    \n",
    "# Create train set\n",
    "X_train, X_test, y_train, y_test = train_test_split(data['overfeat'],data['labels'],test_size=1800, random_state=0)\n",
    "\n",
    "# Create validation and test sets\n",
    "X_valid, X_test, y_valid, y_test = train_test_split(X_test, y_test, test_size=1000, random_state=0)\n",
    "\n",
    "print('Train:', X_train.shape, y_train.shape)\n",
    "print('Valid:', X_valid.shape, y_valid.shape)\n",
    "print('Test:', X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch generator\n",
    "def get_batches(X, y, batch_size):\n",
    "    # Shuffle X,y\n",
    "    shuffled_idx = np.arange(len(y)) # 1,2,...,n\n",
    "    np.random.shuffle(shuffled_idx)\n",
    "\n",
    "    # Enumerate indexes by steps of batch_size\n",
    "    # i: 0, b, 2b, 3b, 4b, .. where b is the batch size\n",
    "    for i in range(0, len(y), batch_size):\n",
    "        # Batch indexes\n",
    "        batch_idx = shuffled_idx[i:i+batch_size]\n",
    "        yield X[batch_idx], y[batch_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a multilayer network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a new graph\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    #create placeholders\n",
    "    X = tf.placeholder(dtype=tf.float32,shape=[None,4096])\n",
    "    y = tf.placeholder(dtype=tf.int32,shape=[None])\n",
    "    \n",
    "    #hidden layer with 64 hidden units (ReLu)\n",
    "    hidden = tf.layers.dense(X,64,activation=tf.nn.relu,kernel_initializer=tf.variance_scaling_initializer(scale=2,seed=0),\n",
    "                            bias_initializer=tf.zeros_initializer(),name='hidden')\n",
    "    \n",
    "    #output layer\n",
    "    logits = tf.layers.dense(hidden,4,activation=None,kernel_initializer=tf.variance_scaling_initializer(scale=1,seed=0),\n",
    "                            bias_initializer=tf.zeros_initializer(),name='output') \n",
    "    \n",
    "    # Get weights/biases of the hidden layer\n",
    "    with tf.variable_scope('hidden', reuse=True):\n",
    "        W1 = tf.get_variable('kernel')\n",
    "        b1 = tf.get_variable('bias')\n",
    "\n",
    "    # Get weights/biases of the output layer\n",
    "    with tf.variable_scope('output', reuse=True):\n",
    "        W2 = tf.get_variable('kernel')\n",
    "        b2 = tf.get_variable('bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    #create training placeholder\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    #apply dropout (TO INPUTS AND HIDDEN UNITS)\n",
    "    inputs = tf.layers.dropout(X,rate=0.5,seed=0.5,training=training)\n",
    "    hidden = tf.layers.dropout(hidden,rate=0.5,seed=0.5,training=training) \n",
    "    \n",
    "    #loss function cross entropy\n",
    "    ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,logits=logits)\n",
    "    \n",
    "    #compute mean of cross entropy\n",
    "    mean_ce = tf.reduce_mean(ce)\n",
    "    \n",
    "    #gradient descent\n",
    "    lr = tf.placeholder(dtype=tf.float32)\n",
    "    gd = tf.train.GradientDescentOptimizer(learning_rate=lr)\n",
    "    \n",
    "    #L2 term FOR W1 AND W2\n",
    "    l2_term_W1 = tf.nn.l2_loss(W1)\n",
    "    l2_term_W2 = tf.nn.l2_loss(W2)\n",
    "    \n",
    "    #regularization strength\n",
    "    alpha = tf.placeholder(dtype=tf.float32)\n",
    "    \n",
    "    #loss function\n",
    "    loss = mean_ce + alpha * (l2_term_W1 + l2_term_W2)\n",
    "        \n",
    "    #training op\n",
    "    train_op = gd.minimize(loss)\n",
    "    \n",
    "    #compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits,axis=1,output_type=tf.int32)\n",
    "    is_correct = tf.equal(y,predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct,dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training the network\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n",
      "Multilayer fully-connected network accuracy (test set): 0.805\n"
     ]
    }
   ],
   "source": [
    "#validation accuracy\n",
    "valid_acc_values = []\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    #initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    #set seed\n",
    "    np.random.seed(0)\n",
    "    \n",
    "    #train several epochs\n",
    "    for epoch in range(150):\n",
    "        #accuracy values (train) after each epoch\n",
    "        epoch_acc = []\n",
    "        \n",
    "        #get batches of data\n",
    "        for X_batch,y_batch in get_batches(X_train,y_train,260):\n",
    "            _,acc_value = sess.run([train_op,accuracy],feed_dict={X:X_batch,y:y_batch,lr:0.001,training:True,alpha:0.001})\n",
    "            epoch_acc.append(acc_value)\n",
    "        \n",
    "        #evaluate validation accuracy\n",
    "        valid_acc = sess.run(accuracy, feed_dict={X: X_valid,y: y_valid,training:False})\n",
    "        valid_acc_values.append(valid_acc)\n",
    "        \n",
    "        #print progress\n",
    "        #print('Epoch {} - valid: {:.3f} train: {:.3f} (mean)'.format(epoch+1, valid_acc, np.mean(epoch_acc)))\n",
    "    print ('Done')\n",
    "    \n",
    "    # Evaluate test accuracy\n",
    "    test_acc = sess.run(accuracy, feed_dict={X: X_test,y: y_test})\n",
    "    print('Multilayer fully-connected network accuracy (test set): {:.3f}'.format(test_acc))   \n",
    "    \n",
    "    # Weights of the hidden layer\n",
    "    weights_hidden = W1.eval()\n",
    "    weights_hidden = W2.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9+PHXOzeEEAIEkPsGOQQkgnggigdaFe+zHvXAWq3U2lptrVq/bW1trforXogXVgt4IVq8oVBAhCD3fZMQIIGckHv3/ftjJnETdpMFs9nAvp+Pxz6yM/PZ2ffObuY9n89n5jOiqhhjjDEAUeEOwBhjTNNhScEYY0w1SwrGGGOqWVIwxhhTzZKCMcaYapYUjDHGVLOk0ISJSHcRURGJcac/FZFbgil7FO/1WxGZ8kPiNU2TiMSLyDoR6RDuWOoiIo+LyL/CHUddROQfIvLTcMcRSpYUQkhEPheRJ/zMHy8ie490B66qF6rqmw0Q1xgRyay17j+r6h0/dN2RTETGisgGESkWkbki0q2OskNF5H8iUiAimSLyqM+yU0XkSxHJFZEcEXlXRE7wWS4i8lcROeA+nhIRqSO0CcB8Vd3bMJ+06fL32/ZT5nERqRCRgz6Pnj7Lh4rIMvd7XCYiQ31e/jfgdyISF6rPEG6WFELrDeAmP/+wNwFvq2pl44cUWY625nQU79MW+AD4PdAaSAem1/GSd4D5btmzgLtF5FJ3WQowGegOdAOKgNd9XjsBuAwYApwEXAzcVcd73QW8dUQf6Pg3XVVb+Dy2Abg7+4+Af+F8D28CH1UlAVXdA2wALg2w3mOfqtojRA+gGVAAjPaZlwKUAkPc6R8By4FCIAN43Kdsd0CBGHf6v8Ad7vNo4O/AfmAbcE+tsj8B1uPsULYBd7nzE4ESwAscdB8dgceBf/m896XAWiDffd8TfZbtAH4FrHI/33QgIcA26AXMAQ64sb4NtPJZ3gVnZ5rjlpnks+xOn8+wDjjZna9Ab59ybwB/dJ+PATKB3wB7cXaGKcAn7nvkuc87+7y+Nc5ON8tdPtOdvwa4xKdcrPsZhvr5nBOART7TVdu5f4DtUgwM8Jl+F3g4QNmTgSKf6UXABJ/p24HFAV7b1Y0jptb2egH41P3+FwIdgGfdz78BGOZTviPwvrv9tgP3+SwbAXzj/k72AJOAOJ/lCvwU2Oyu+3lAAsT6ODV/g++632EBTgId6LPsIvc3UQTsxvk9+v1t1/c+tZad765PfObtAsb5TP8OeD3c+5dQPaymEEKqWgLMAG72mX0NsEFVV7rTh9zlrXASxN0iclkQq78T5whxGJAGXFVreba7vCVOgnhGRE5W1UPAhUCWfn+UlOX7QhHpC/wb+AWQCswGPq5VZb4GGAf0wDlavTVAnAI8ibNjOREnCTzuvk80zg56J04C7ARMc5dd7Za72f0Ml+IkjWB0wNnRd8PZWUfh7PS78f1OcpJP+beA5sBAoB3wjDt/KvBjn3IXAXtUdYWf9xwIVH2nuNt5qzvfn2eBm0UkVkT6AaOArwKUHY2ToP2+l/s80PsMBrbp4bXSa4BHgLZAGc6O/Tt3+j3gHwAiEgV87L5HJ2As8AsRucBdjwe4333dKHf5z2q918XAKTg1m2uACwjOp0AfnO/kO5wDiiqv4hzoJAGDgDnB/LZ9XOI2z60Vkbt95g8EVqm793etoub2Xe9+luNTuLPS8f4AzsA50mnmTi8E7q+j/LPAM+7z7gSuKcwBfurzuvN9y/pZ70xgovt8DJBZa/njuEdPOE0gM3yWReEcPY1xp3cAP/ZZ/hTwUpDb4zJguft8FM7R52ExA59XxetnWX01hXIC1FzcMkOBPPf5CThHlil+ynXEORJt6U6/BzwYYJ2vAn+pNW8hcGuA8qcBW4BK9/P8IUC5k4Bc4EyfeR58aiA4O07FzxE4cCO1ahHu9nrFZ/rnwHqf6cFAvvt8JLCr1usfJsCRMs6BxIe1vqszfKZnAA8FeG31b9DPslbuupLd6V04zWIta5U77LftZ10D3O822v0e9gDX+/z2p9Uq/zY1a/Dn4STasO9fQvGwmkKIqeoCnB3feLcz6xSc9mQARGSk2ymZIyIFOFXttkGsuiNOc1OVnb4LReRCEVnsHg3l4xzlBrPeqnVXr09Vve57dfIp49tpWQy08LciEWknItNEZLeIFOK01VbF0QXYqf77VrrgHGkfjRxVLfWJobmIvCwiO90Y5gOt3JpKFyBXVfNqr0Sdo8yFwJUi0grnKPTt2uVcB3FqNL5a4iSVGkSkNfAZ8ASQ4MZwgYj8rFa53jhHyxNV9X91vFdL4KC6e6xa8oAkP/P3+Twv8TNd9X12AzqKSH7VA/gt0N6Nsa+IfOKeOFEI/JnDf2dB/VZ8iUi0iPxFRLa6693hLqpa95U4v+mdIjJPREbVt84qqrpOVbNU1aOqi4Dn+L6mHcz3mITTXHZcsqTQOKbiNIPcBHyhqr7/gO8As4AuqpoMvITT5FKfPTg7kypdq56ISDxOG/Dfgfaq2gqnCahqvfUNjZuFszOoWp+477U7iLhqe9J9v5NUtSVOc0xVHBlA1wCdwRk4/RH+FOM091Spfapl7c/3ANAPGOnGMNqdL+77tHZ3+v686cZ8NfCNqgbaBmvxaVIQkUQ3/rV+yvYEPKo6VVUrVTUTp9nsIp/Xd8NpTvo/Va3dSVzjvdzn/t4HnKaPnj+gwz0D2K6qrXweSapaFeuLOH0Qfdxt+1uC+/3W5wZgPHAukIxTa6Zq3aq6VFXH4zQtzcSpgUD9v21/lO9jXgucVOvkkJOouX1PpGbz3XHFkkLjmIrz474TZyfjKwnnSLVUREbg/DMEYwZwn4h0FpEU4CGfZXFAPE4NpVJELsRpXqqyD2gjIsl1rPtH7imWsTg71TKcDs4jlYRz9JUvIp2AX/ssW4KT3P4iIokikiAip7vLpgC/EpHh7imYvX1O8VwB3OAeTY7DOXunvhhK3BhaA49VLVDnbJJPgRdEJMVt4x/t89qZOB29E3G+x0A+BAaJyJUikgA8itM2vcFP2U04ufYGEYkS5/qBa3F3NO52mgM8r6ov+Xn9VOCXItJJRDrifD9v+AvKTTibcTqEj8YSoFBEfiMizdxtPkhETnGXJ+GcJHFQRPoDdwdc05FJwvnNHcA5APhz1QIRiRORG0UkWVUr3Pf3uIvr+21XnRKe4v6uRgD34ZxxBE4TrQfnfyteRO5158/xWcVZOL+Z45IlhUagqjtwdqiJOLUCXz8DnhCRIpwdyQyC8wpOu/tKnE64D3zerwjnhz4Dp/ngBt/3dXdU/wa2uU0CHWvFuxHn6PifOGfbXIJzFk55kLH5+gPOTrUA+E+tOD3uunvjtBFn4uwcUdV3gT/h1KSKcHbOrd2XTnRfl4/TZj6znhiexTkTbD+wGKfpxtdNQAXOEW82Trt4VYwlOLWuHr6x16aqOThNGn/C2eYjgeuqlovISyLyklu2ELgCp4M2DyfJrXFfC3AHTm3iMfE5l97n7V7G6fxd7b7uP+68QF52P+MR8/mOhuKcebQfJ2FX7XR/hfP7KsL5TdZ1Gu6RmIrThLkb5yyjxbWW3wTscJuWfop7QkB9v23XdTj9OUXu+/xV3et/3N/4ZTg1+3zgNuCyqt++ONeLDKD+39wxS/w3QxpjqohzYVlfVf1xvYWbILc5cTkw1q0ZmaMkIk8DW1X1hXDHEiqWFIypg9vctBy4SVXnhzseY0LNmo+MCUBE7sTpaP3UEoKJFFZTMMYYU81qCsYYY6o1ymBhDalt27bavXv3cIdhjDHHlGXLlu1X1dT6yh1zSaF79+6kp6eHOwxjjDmmiMjO+ktZ85ExxhgflhSMMcZUs6RgjDGmmiUFY4wx1SwpGGOMqWZJwRhjTDVLCsYYY6pZUjDGmFrmbshmXVZhveXWZhUwd2N2yOIoq/Twr8U7yS4srb9wA7GkYIwJaE9BCRf/83/8/fONFJf7u2tq+Hm9yoqMfLxeZxy31ZkFjJ+0gNWZBUe1vq05B7lzajoPvLuSusaGU1V+OX0lE6ams2P/oaDWPW9TDuf8/b88+N5Kvli7tzpmf/KLy7lpyhIembmGy55fyKZ9h93ZNSQsKRhjqu0rLOWh91ex84Czk/vrpxtYv6eISXO3MOZv/+WmV7/llteWsDIj+FsUF5dXUuHxAs6OdMr/tjHlf9uql89evYd/fLEx4A5SVSkp9/hdBvDc15u57PmFTJy+gm05B7ntzaWszCzgkY/W+F2nx6v88ZN1fLvtgN/1/ek/66n0Kuv3FLJke27A9/1m6wE27iuiwqP8afZ6AGYu383DH6wiM6/4sPJllR5+P3MNBSUVfLpmLxPeWsZlLyzki7V7eX7uFn45YwVbsp0d/64DxVzx4iJWZOTz4Lh+VHqVK19cxKIt+wPG01COuWEujDGhoar8+r1VzN+Uw7fbc3n04gHMXJHFvWf35uz+qUyas4WCkgo2Zx/k/z5Zx7s/HYWIoKqs21PIgs37aZ0Yx1n9UikormDuxmzmbMgmfUcebVvE89CF/Zm/KYcPlju3ue7TPolOrRK4f/oKyiq9eFT59QX9a8RUUu7hF9OXs3DLAWbeczq927Ugu7CUj1ZkceXwzmTkFjNp7hb6d0ji45VZfLp6D83iornn7F48P3crM1fs5oqTO9dY59fr9zFlwXamLc3gvbtH0b9Dy+pl8zblMGdDNvef25fXF23njUU7GNmzjd/t9fqiHbROjOOmU7vx3Neb+fm/l/PxyiwAPvhuN/ee3Zt7zu5NVJRzu+fXFuxgV24xb90+glE92/Dxqiz+8ukGJry1DICE2Ci+WrePX1/Qj2e/2kylV/nXHSMZ0aM144d2YsLU9KO6AfWROuaGzk5LS1Mb+8iY4BSXVzJnQzYer5JdWMa8TTnszi/hzjN7cu0pXYiO+v7+9P9avJNHZq7h+hFdeX9ZJhVeL6kt4pn7qzEkxn9//Dj1mx08+tFa3rlzJH3bJ3HjK9+yMUDTRv8OSYzum8qirftZs9tpo584tg+zV++hsLSC1KR4dueVcGafVGatzOK+sX3olZoIgKqz412VmU/z2Gh6tWvB1NtGcN3kxWzYW0Rys1haxMfgVeWzX4xm/qYc/v7FRv58+WBG9WzD5S8sZG9hKXMeqBn/Da8sZmuOc3fTaBF+Pa4fXi98tyuPT9fsJSkhhi/uH82zX23m5XlbmXTDybyxaAd927fgj5cNBiAjt5jRf5vLPWN6c+85vTn3H/PIzCvhmrTO3Ht2H/762Qb+s3oPFw7qwDPXDmXHgUNc9eI3nNqzNVNuOaU6lkNllSzedoDBnZIpq/Ry6+tL2JpziC6tm/H6rSPo3a5FdVmvV6sTzNEQkWWqmlZvOUsKxjS8Q2WVNXZEvkorPDz60RrGntieCwZ2ACDvUDlJCTHERPtv0X1l/jaax0dz48huQcegqtz6+lLmbcqpnte3fQuaxcWwMiOfHm0TSW0RX71s1e58Tunemqm3jeDjVXv45fQVPH3NEMYP7XRY/KOfmkv3NomUe7xs2FvIY5cMZGz/dmQXlTF/cw4tE2I5u387OrVqBjhNNh+t2E2L+BjOH9iB1ZkFXP7CQiq9yos3nsy5A9pzx5vpNWIF5+j5ueuG4fEqP3v7O1KT4sk9VM4T4wfy6eq9fLPtAG/+ZARn9Gl72OdftjOXq176hpO7pvDKzWm0Toxj494iLnh2Pg+O68foPqlcN3kxB8ucvpLmcdGc3rstE8f2YVCnZHbnlzD6qbl4vEpcTBTllV6mTziVkT3b8MjM1UxbksGC35xDh+QE1uwuYMPeIq48uVN17enVBdv50+z1JMbFcLCskoTYKD6dOJoebRMDfmcFxRW8s2QXVw3vTGpSfMByR8OSgjFhMmNpBr/9cDVvBNhZ/fGTdUxZsB0R+M24/uQXV/Dagu2c1DmZV25OIyUxrkb57MJSTv/rHOKio1j827EkJcQGfO+VGfks3ZHLjSO78cHyTH734RoeurA/5w9oT4v4GNq1TEBV+WTVHmakZ1Dp+f7/v1XzWB67ZCAdkhMAp5bRPM5/Ynt1wXb+75N1iMCLNw5n3KAOR7yd3l+Wyd7CUu45uzfgHAnvzC2u0bnbJjGe5ObO5/3FtOXMXJHFk1cM5voRXVFVCksrSW4WeHvMXr2HX0xfQcfkBO4/ry//3ZjD7NV7+ObhsbROjKOgpIIDB8sA6JTSjPiY6Bqvnzx/KwdLK7lpVHfGT1pASmIct4zqzoPvr+LmUd14YvygOj/jF2v38smqPZzSPYVzB7TnhORmR7ydGoolBWOCtK+wlNxD5Zx4Qsv6C+PsaB58bxVllR6Sm8Xyt6uGcHb/dgAs3LKfW15bQqVXGT+0I89dN6zGa7/ZeoAbpizm2rQu1R2OAOcNaM+8TTl0atWMEd1b87/NOVwwqAOPXTKQZ77cxHNfbwbgsUsGcOtp3fnVu6tYsuMAo/ukcumQjozs2YYt2Qe54oWFFJZW0jE5gfySCoZ3S2HqbSMQOfpmB39Kyj3c+voSLj7pBG4a1b1B1x1IWaWHjXuLOKlzqyN63bKdudzz9nL2uqd1XpvWhb9eddIRv/9HK3YzcdoKAE7v3YY3fjKC2AA1u6bIkoJp0lSV7KIy2rdM8Ls891A5D72/it9fPIAurZuHLI7i8kp+9P8WkJVfwn/uO7NGG24Vj1fJPVROalI8WfklXPDsfLqkNGdMv1Tmbsxh074i7j+3D0VllbyzeBcdWzWjb4ckvl6/j2WPnEe5x8uD761kb0EpOw4Uk9I8ltkTzyQhJpppSzMY0LElQ7u0In1HLhPeWkaFx0vX1s1Zm1XISz8+mUdmruGkzq3ILy4n91A5Pz+nDw+8u5IhnZPZnH2Q4nIP5w1oz4a9hZSUe3j80oG8MHcrewtL+c99Z4T16LSp8HqVVbsLWLL9AJcN7US7AL+7uqgqP371Ww4cLGf6XaPqrKE0RZYUTJM25X/bePLTDcx9YAxd2xy+03/uq80889Um7j+3LxPP7YPHq0yas4UOyfGM6dcuYDLxNX3pLuJjorl0SMeAHXS/n7mGtxbvpEV8DL1SE3n/7tMOa9efPH8rT366gWuGd2FXbjErM/P5dOKZdGuTyKGySu595zvmbswhJkoY1asNf7nyJHYeOMQNr3zLpBuGsXxXPq8t3M7oPqnEx0Txi3P7MqCj/1pJpceLAl5Vxk9ayJbsg1R6lam3jSC/pIL7/r2cmChhWNdWTJswigqPl9cWbmfSnC14vMr0u0YxtEsrvF6l3OMlITba7/uYo1Pp8SIiNTrojxXBJgU7JdU0uv0Hy3juq814vMq8zTnc1KYba3YXcOvrS3j5puEM7tSKf33r3CTqv5uymXhuHxZu2c8zX22qXseE0T15aFz/gDv7d77dxW8/XA04Z7BcMawTUQKDOiUzrGsKAF+u28dbi3dy+xk9OLlrCve88x3PfLWJX53fr0Zzy6yVWaQ0j+P97zKp9Cp/vnww3do4nYWJ8TG8cnMaa7IK6ZWaWN3e36FlAqlJ8Uyev411WYVcm9aFv1xZf5OFb0L6xzVDGf/8AnqlJnJmn7ZUepX2LeMpKq3k6auHEh0lREdF87Mxvbl6eBeKSivomerUdKKihIQoSwgNLdCJAMeTkCYFERkHPAdEA1NU9S+1lncF3gRauWUeUtXZoYzJhN/TX2ykpMJDSvNYFm7ez02nduPD5bvZf7Cc+6ev5Kdn9SKnqIy0biks25VH7qFyPl6ZRVJ8DO/ceSpvf7uTyfO3sTuvhJ+c3p2oKGFgx5bVnYTzN+Xw+4/WMKZfKhef1JG/fraBx2atrX7/S4Z0JC46ive/y6Rf+yR+fUE/EmKj+Wp9J56fu5X0HXk8fulATjyhJbvzS1izu5CHLuzPeQPas2JXPlecXPNsnJjoKIZ2qdnOHR0l/GjwCbyxaAct4mN44Px+R7ydBnRsyb9uH0nrxDhEhNho4dVbTsHj1cNqV6lJ8Q1+toqJTCFLCiISDTwPnAdkAktFZJaqrvMp9ggwQ1VfFJEBwGyge6hiMo1nze4CvtuVx821OiHXZhUwbWkGPzmtBwfLKvh87T48XuWLdXvp0TaRHQcO8cjM1fRMTeS3PzqRK15YxFfr9/HZ2r1cMKgDgzsn82SnwfRu14I//mc9/1m9B4CTOicz5ZY0VuzK575py+nTrgWTbjiZFvExjB/akYKSCjxe5e1vd/HyvK2owk/P6sW95/SubmL5+9VDGN4thae/2MhVLy5i7q/G8OVapyP4goEd6NE2kV6ph/c5BHLZsE68sWgH943tfdQ77NoXTg3qlHxU6zEmWKGsKYwAtqjqNgARmQaMB3yTggJVjavJQFYI4zGNJL+4nNvfXMq+wjJO7ppSvSMrq/TwwIyVtEmMY+LYPvx3UzYz0jN5f1kmGbkl/OWKwWzbf4jJ87dx62ndGdq5Fa0T4/jHF5soKq3kkiEdARAR7jizJ6P7prK3oJSs/BL+8PE6LnpuAQcOlXFS51ZMuTmNFu51ArHRUbR1z8f/5Xl9uXlUN7xePayzMTpK+PGp3Tijd1vOf2Y+T32+kd15JfRp16LOc8sDGdqlFV/cP5o+fjqvjWmqQpkUOgEZPtOZwMhaZR4HvhCRnwOJwLn+ViQiE4AJAF27dm3wQCNVcXkl97z9HTeO7Ma5A9r7LfO7D1ezaKszRsxlQzsx8dw+1csqPV6e/HQDOw8c4sFx/enbPgmA33+0lgMHy2kWG83rC3fw9DVDAKfzeMPeIqbcnEZy81hO6+Wcw//U5xsQgbEntueKZrGc3DWFsSe2IypKOKtvKh8u303rxDhO61XzqLlv+6Tq9xzYMZk7p6ZzwQDnCtJmcYHb09u2qPuovXvbRH5yRndenreNKIG7x/Sqs3xdquIz5lgRyqTgrwew9qlO1wNvqOrTIjIKeEtEBqmqt8aLVCcDk8E5+ygk0UagJ2dvYO7GHPJLKqqTgser1WdWbN5XxNvf7mJ4txTKK71MmruZq9I606lVsxpn3TSPi+bC5/7HWX1TUVXmbszhgfP6knOwjGlLMnj4ov5s2FPES/O2cm1al+r3Sk2Kp3+HJDbsLSKtW0p1E4vvhVBj+jlJ4cJBHeo8J3xw52QWPXTODxoGwNe9Z/fm/WWZ7D9YzvkDjvzCLGOOVaHsSs8EuvhMd+bw5qHbgRkAqvoNkAAcfgmoaXDzNuXw1uKddGndjOW78tm8r4ji8krOf2Yej360BoA3Fu0gLiaKV25O46WbhgPw0n+3Ulrh4ZbXljBvUw5/vGwQC35zDj8e2ZWM3GIy80q4Ylgn7h7Ti1tO6065x8v901dw2xtL6ZXagkcuPrFGHKf3dr7u8wf6r6mc3b8do/umHtY34U9DJQSApIRY/njZYH40+AQGWzu+iSChrCksBfqISA9gN3AdcEOtMruAscAbInIiTlLIwYTMN1sP8PnavcxcsZs+7Vrwxm0jOOupuby7LJOEmCi25hxia84hBndK5oPvdnPZ0I60dodduGp4Z6YvzSArv4T0nXn88/ph1e38f/BzuX+v1Bac1TeVeZtyGNmjNZNvSjtsiIaLBp/ARyt2c9HgE/zG2zIhlqm3jWjgrRCccYM6HNXwDcYcy0J68ZqIXAQ8i3O66Wuq+icReQJIV9VZ7hlHrwAtcJqWHlTVL+pap128dnS8XuWpzzfy0rytJMRGcVqvtvz2ov70bpfEhKnpLN2RS3G5h7EntmPH/mLW7XFGtJx935nVF1pl5BYz5u//xeNVfjOuf1Bt7VtzDvLZmr3ccWaPw8aVMcY0Hrui2ZB7qJzpSzMoLq9kbVYhczZkc8PIrjx68YAaV7p+tW4fd0xNJyE2ijkPjHHO9PnnAoZ1bcX0u0bVWOfk+VspLvcwcWyfBh9PxxgTOnZF83Eg91A5S7YfQNUZwbH2QGAl5R7WZBWQ1i0FEeHbbQd46IPVDOzYkp5tE3lj0Q4KSyuJEoiLieKhC/tz1+ieh+3Mx/RLZWDHlowf2pGO7lDHM+85nXYtDz9LZ8Looz8TxxjT9FlNoQmq8Hh5c9EOnvt6M0Wl398Xd9INw7j4pI7VQx8/OXs9WQWlXJPWmdvP6Mk1L39Ds9hoKr3K/oNlnNG7LY9eMsBOizTGWE3hWDVvUw5PfLyWrTmHGN03lfvO6U1ifAy/n7mGX85YSUm5h3eXZbJkey4DTmjJuQPaM/WbnXzw3W6Sm8Uy465RdE5p5o5AGm9NPMaYI2JJoYnYsf8Qf/zPOr5an033Ns159ZY0zunfrnqnPvnmNK54YSG/fm8VrRPj+PPlg6tvpzioYzIvztvK09cMqR4Tp+pGKcYYcySs+agJePvbnfxh1jpio4Wfj+3DT07v7vdMnYzcYj5ds4dr07pW343KGGOCYc1Hx4i1WQU8Pmstp/Zsw9NXD6nz5h9dWje3jl5jTEhZUgijskoPv5y+klbN4/h/1w077N68xhjT2CwpNKLcQ+V8uW4vczfkkFVQwsHSSrbtP8Trt55iCcEY0yRYUmgkKzPyuf3Npew/WM4JyQn065BEm8Q4fnxqt+qbvhtjTLhZUgiRguIKrn9lMUVlFZzaow2frNpDmxZxzLzndIZ0TrZTRY0xTZIlhQaUlV9Cs9hoEuNjuPvtZWzOLuL03m35ZNUeTjwhiZdvSrNbJhpjmjRLCg1kRUY+103+hvJKLx1bNSMzr4Snrx7ClcM717hHgTHGNGWWFBpARm4xd7y5lNSkeC4f1pkFm3O4cWQ3rhzeGcASgjHmmGFJ4QcqKKngtjeWUl7pZdqEUfRu14Jfntc33GEZY8xRsaTwA1R4vPzs7WXsOHCIqbeNpLfdoN0Yc4yzpPAD/H7mGhZuOcDfrx7CqFo3lTfGmGNRKO/RfFzbsLeQaUszuGt0T65y+w6MMeZYZ0nhKL2bnklstHDXWTYWkTHm+GFJ4SiUV3r5cPluzhvQvvqm9sYYczywpHAU5mzYR+6hcq5O6xLuUIwxpkFZUjgKM9Iz6dAygdF9UsMdijHGNChLCkcgv7icRz9aw9yN2Vw5vJNdlGaMOe7YKalBKiip4Pxn5rP/YBmE8Ud7AAAVKElEQVQ3ndqNn43pHe6QjDGmwVlSCNLS7blkF5Ux5eY0zh3QPtzhGGNMSFjzUZDSd+YREyWc0adtuEMxxpiQsaQQpO925jGwUzIJsdHhDsUYY0LGkkIQyiu9rMzMJ61bSrhDMcaYkLKkEIQ1WQWUVXotKRhjjnuWFIKwbEceAMMtKRhjjnOWFIKwbGceXVo3o13LhHCHYowxIWVJoR6qSvrOPNK6tQ53KMYYE3KWFOrx4fLd7D9YZk1HxpiIYBev1eH5uVv42+cbGdmjNZcP6xTucIwxJuQsKQSwLquQv32+kYtPOoGnrxlCfIxdn2CMOf5Z81EAW3MOAnDvOb0tIRhjIoYlhQAy8ooB6JzSPMyRGGNM47GkEEBmXgkpzWNpEW8tbMaYyGFJIYCM3GK6tLZagjEmsoQ0KYjIOBHZKCJbROQhP8ufEZEV7mOTiOSHMp4jkZlXQhdrOjLGRJiQtY2ISDTwPHAekAksFZFZqrquqoyq3u9T/ufAsFDFcyS8XmV3Xgnn230TjDERJpQ1hRHAFlXdpqrlwDRgfB3lrwf+HcJ4gpZdVEa5x0tnaz4yxkSYUCaFTkCGz3SmO+8wItIN6AHMCbB8goiki0h6Tk5OgwdaW9WZR11SmoX8vYwxpikJZVLwd1d7DVD2OuA9VfX4W6iqk1U1TVXTUlNTGyzAQDLtdFRjTIQKZVLIBLr4THcGsgKUvY4m0nQEkJFbAkBnqykYYyJMKJPCUqCPiPQQkTicHf+s2oVEpB+QAnwTwliOSGZeMe2S4u3Wm8aYiBOypKCqlcC9wOfAemCGqq4VkSdE5FKfotcD01Q1UNNSo8vILbFagjEmIoX0cl1VnQ3MrjXv0VrTj4cyhqORkVdsQ2UbYyKSXdFcS6XHy56CUqspGGMikiWFWvYUlOLxql3NbIyJSJYUalm3pxCAbm0SwxyJMcY0PksKtXzwXSZtW8SR1t36FIwxkceSgo/9B8v4en02lw/rRGy0bRpjTOSxPZ+Pmct3U+lVrk7rUn9hY4w5DllScKkqM9IzGNqlFX3bJ4U7HGOMCQtLCq61WYVs2neQa6yWYIyJYJYUXFuyDwIwsmfrMEdijDHhY0nBlV1UCkC7pPgwR2KMMeFjScGVU1RGs9hoWsSHdOQPY4xp0iwpuLKLykhNikfE320gjDEmMlhScGUXllnTkTEm4gWVFETkfRH5kYgct0kku6iUdi0tKRhjIluwO/kXgRuAzSLyFxHpH8KYwiK7qIx2SQnhDsMYY8IqqKSgql+p6o3AycAO4EsRWSQiPxGR2FAG2BhKKzwUlVaSas1HxpgIF3RzkIi0AW4F7gCWA8/hJIkvQxJZI8opKgOwpGCMiXhBnX8pIh8A/YG3gEtUdY+7aLqIpIcquMZi1ygYY4wj2JPyJ6nqHH8LVDWtAeMJi+xCp6ZgfQrGmEgXbPPRiSLSqmpCRFJE5GchiqnRZbvNR3b2kTEm0gWbFO5U1fyqCVXNA+4MTUiNL7uolOgooXXzuHCHYowxYRVsUogSn0t9RSQaOG72oDlFZbRtEUdUlF3NbIyJbMH2KXwOzBCRlwAFfgp8FrKoGpldo2CMMY5gk8JvgLuAuwEBvgCmhCqoxpZdWMYJyZYUjDEmqKSgql6cq5pfDG044ZFdVMaQLsnhDsMYY8Iu2OsU+gBPAgOA6kNqVe0ZorgaTaXHy4FDZaRa85ExxgTd0fw6Ti2hEjgbmIpzIdsxL/dQOap24ZoxxkDwSaGZqn4NiKruVNXHgXNCF1bjybYhLowxplqwHc2l7rDZm0XkXmA30C50YTUeG+LCGGO+F2xN4RdAc+A+YDjwY+CWUAXVmKqHuGhpfQrGGFNvTcG9UO0aVf01cBD4ScijakTVzUctrKZgjDH11hRU1QMMl+P05sXZRaWkNI8lLua4vamcMcYELdg+heXARyLyLnCoaqaqfhCSqBpRTlGZdTIbY4wr2KTQGjhAzTOOFDjmk4INcWGMMd8L9orm46ofwVd2YRkjeySGOwxjjGkSgr2i+XWcmkENqnpbg0fUiFTVaT6y+ygYYwwQfPPRJz7PE4DLgayGD6dxFZRUUO7xWvORMca4gm0+et93WkT+DXwVkogaUY5dzWyMMTUc7XmYfYCuDRlIOFTfhtOSgjHGAEEmBREpEpHCqgfwMc49Fup73TgR2SgiW0TkoQBlrhGRdSKyVkTeObLwfxgb4sIYY2oKtvko6UhX7F4J/TxwHpAJLBWRWaq6zqdMH+Bh4HRVzRORRh1PyYa4MMaYmoKtKVwuIsk+061E5LJ6XjYC2KKq21S1HJgGjK9V5k7geVXNA1DV7OBD/+Gyi8poHhdNi/hg+9uNMeb4FmyfwmOqWlA1oar5wGP1vKYTkOEznenO89UX6CsiC0VksYiM87ciEZkgIukikp6TkxNkyPWzq5mNMaamYJOCv3L1HV77Gyup9rUOMTid1mOA64EpItLqsBepTlbVNFVNS01NDSLc4GQXlVp/gjHG+Ag2KaSLyD9EpJeI9BSRZ4Bl9bwmE+jiM92Zw69tyAQ+UtUKVd0ObMRJEo3Chrgwxpiagk0KPwfKgenADKAEuKee1ywF+ohIDxGJA64DZtUqMxPn9p6ISFuc5qRtQcb0g+UUWvORMcb4Cvbso0OA31NK63hNpXuXts+BaOA1VV0rIk8A6ao6y112voisAzzAr1X1wBF9gqNUUu6hqKySdjbEhTHGVAt27KMvgavdDmZEJAWYpqoX1PU6VZ0NzK4171Gf5wr80n00qu+vUbDmI2OMqRJs81HbqoQA4J5Cekzfo9mGuDDGmMMFmxS8IlI9rIWIdMfPqKnHEhviwhhjDhfsVVu/AxaIyDx3ejQwITQhNY4Dh8oBaNMiLsyRGGNM0xFsR/NnIpKGkwhWAB/hnIF0zCoodpJCq2aWFIwxpkqwHc13ABNxrjVYAZwKfEPN23MeU/KKK0iMiyYu5mgHijXGmONPsHvEicApwE5VPRsYBjTceBNhkF9cQavmVkswxhhfwSaFUlUtBRCReFXdAPQLXVihl19cTqvmseEOwxhjmpRgO5oz3TGJZgJfikgex/jtOPNLKiwpGGNMLcF2NF/uPn1cROYCycBnIYuqEeQXl9O/Q8twh2GMMU3KEd9IQFXn1V+q6XP6FKymYIwxviLy1BtVteYjY4zxIyKTwsGySjxetWsUjDGmlohMCvnFFQBWUzDGmFoiPClYTcEYY3xFZFLIqxriwmoKxhhTQ0QmhfwSp6aQYknBGGNqiMikUDUYXrJ1NBtjTA0RmRTy3D6F5GZWUzDGGF8RmRTyiytoER9jI6QaY0wtEblXzC8pt1qCMcb4EZlJwYa4MMYYvyI0KZSTYtcoGGPMYSI0KVSQbDUFY4w5TGQmhZIKWlmfgjHGHCbikoLXq9Z8ZIwxAURcUigqq8SrNsSFMcb4E3FJocAuXDPGmIAiLinklzhDXFjzkTHGHC7ikkKe3UvBGGMCirikkG/DZhtjTEARlxQK3WGzW1qfgjHGHCbikkJxuQeAxLiYMEdijDFNT8QlhZIKJykkxEaHORJjjGl6Ii4plFZ4iYuOIjpKwh2KMcY0ORGYFDwkxEbcxzbGmKBE3N6xpNxDszhrOjLGGH8iLylUeGhm/QnGGONXRCYF62Q2xhj/QpoURGSciGwUkS0i8pCf5beKSI6IrHAfd4QyHqjqU7CkYIwx/oTsZH0RiQaeB84DMoGlIjJLVdfVKjpdVe8NVRy1lVrzkTHGBBTKmsIIYIuqblPVcmAaMD6E7xeUkgrraDbGmEBCmRQ6ARk+05nuvNquFJFVIvKeiHTxtyIRmSAi6SKSnpOT84OCKim3moIxxgQSyqTg7+owrTX9MdBdVU8CvgLe9LciVZ2sqmmqmpaamvqDgiqt8FqfgjHGBBDKpJAJ+B75dwayfAuo6gFVLXMnXwGGhzAewC5eM8aYuoRy77gU6CMiPUQkDrgOmOVbQERO8Jm8FFgfwngAu07BGGPqErKzj1S1UkTuBT4HooHXVHWtiDwBpKvqLOA+EbkUqARygVtDFY8bk3U0G2NMHUI6frSqzgZm15r3qM/zh4GHQxmDr7JKL6o2QqoxxgQSUY3rZRVewJKCMcYEElFJoepeCtanYIwx/kVmUoiLqI9tjDFBi6i9Y0m51RSMMaYukZUU7FacxhhTp4hKCmWWFIwxpk4RlRSso9kYY+oWmUnBLl4zxhi/IispWEezMcbUKaKSQmmlc/FavA2IZ4wxfkXU3rHUagrGGFOniEoKdkqqMcbULeKSQmy0EBsdUR/bGGOCFlF7x5Jyj9USjDGmDhGVFMoqLSkYY0xdIioplJTbXdeMMaYukZUU7FacxhhTpwhLCl4S7GpmY4wJKKKSQmmFh2Z24ZoxxgQUUXvI0grraDbGmLpEVFKwjmZjjKlbZCUF62g2xpg6RVRSKK3wWEezMcbUIcKSgpeEGEsKxhgTSMQkBVV1mo/iIuYjG2PMEYuYPWSFR/F41foUjDGmDhGTFGzYbGOMqV/EJIUyuz+zMcbUK2KSQnVNwTqajTEmoIhLClZTMMaYwCInKdj9mY0xpl6RkxSso9kYY+oVMUmhrMILWPORMcbUJWKSwvc1hYj5yMYYc8QiZg9pfQrGGFO/yEkKFZYUjDGmPhGTFEqrmo+sT8EYYwKKmKTQtXVzLhzUwWoKxhhTh5hwB9BYzh/YgfMHdgh3GMYY06SFtKYgIuNEZKOIbBGRh+ood5WIqIikhTIeY4wxdQtZUhCRaOB54EJgAHC9iAzwUy4JuA/4NlSxGGOMCU4oawojgC2quk1Vy4FpwHg/5f4PeAooDWEsxhhjghDKpNAJyPCZznTnVRORYUAXVf2krhWJyAQRSReR9JycnIaP1BhjDBDapCB+5mn1QpEo4BnggfpWpKqTVTVNVdNSU1MbMERjjDG+QpkUMoEuPtOdgSyf6SRgEPBfEdkBnArMss5mY4wJn1AmhaVAHxHpISJxwHXArKqFqlqgqm1VtbuqdgcWA5eqanoIYzLGGFOHkCUFVa0E7gU+B9YDM1R1rYg8ISKXhup9jTHGHD1R1fpLNSEikgPsPMqXtwX2N2A4oWAxNgyLsWE09RibenzQdGLspqr1dsoec0nhhxCRdFVt0n0WFmPDsBgbRlOPsanHB8dGjL4iZuwjY4wx9bOkYIwxplqkJYXJ4Q4gCBZjw7AYG0ZTj7GpxwfHRozVIqpPwRhjTN0iraZgjDGmDpYUjDHGVIuYpBDsvR0ak4h0EZG5IrJeRNaKyER3fmsR+VJENrt/U8IcZ7SILBeRT9zpHiLyrRvfdPeK9XDG10pE3hORDe62HNUEt+H97ne8RkT+LSIJ4d6OIvKaiGSLyBqfeX63mzj+n/v/s0pETg5jjH9zv+tVIvKhiLTyWfawG+NGEbkgXDH6LPuVe6+Ytu50WLbjkYiIpBDsvR3CoBJ4QFVPxBn76R43roeAr1W1D/C1Ox1OE3GuSq/yV+AZN7484PawRPW954DPVLU/MAQn1iazDUWkE849Q9JUdRAQjTPsS7i34xvAuFrzAm23C4E+7mMC8GIYY/wSGKSqJwGbgIcB3P+d64CB7mtecP/3wxEjItIFOA/Y5TM7XNsxaBGRFAj+3g6NSlX3qOp37vMinJ1ZJ5zY3nSLvQlcFp4IQUQ6Az8CprjTApwDvOcWCXd8LYHRwKsAqlquqvk0oW3oigGaiUgM0BzYQ5i3o6rOB3JrzQ603cYDU9WxGGglIieEI0ZV/cIdRgecMdM6+8Q4TVXLVHU7sAXnf7/RY3Q9AzyIz+jQhGk7HolISQr13tsh3ESkOzAM5w507VV1DziJA2gXvsh4FueH7XWn2wD5Pv+U4d6WPYEc4HW3iWuKiCTShLahqu4G/o5zxLgHKACW0bS2Y5VA262p/g/dBnzqPm8yMbrju+1W1ZW1FjWZGAOJlKRQ570dwk1EWgDvA79Q1cJwx1NFRC4GslV1me9sP0XDuS1jgJOBF1V1GHCI8De31eC2y48HegAdgUScZoTamsxv0o+m9r0jIr/DaYJ9u2qWn2KNHqOINAd+Bzzqb7GfeU3qe4+UpFDfvR3CRkRicRLC26r6gTt7X1WV0v2bHabwTgcuFed+F9Nwmjuexanyxrhlwr0tM4FMVa26x/d7OEmiqWxDgHOB7aqao6oVwAfAaTSt7Vgl0HZrUv9DInILcDFwo35/sVVTibEXzgHASvd/pzPwnYh0oOnEGFCkJIU67+0QLm77/KvAelX9h8+iWcAt7vNbgI8aOzYAVX1YVTu797u4DpijqjcCc4Grwh0fgKruBTJEpJ87ayywjiayDV27gFNFpLn7nVfF2GS2o49A220WcLN79sypQEFVM1NjE5FxwG9w7r9S7LNoFnCdiMSLSA+cztwljR2fqq5W1XY+94rJBE52f6tNZjsGpKoR8QAuwjlTYSvwu3DH48Z0Bk7VcRWwwn1chNNu/zWw2f3bugnEOgb4xH3eE+efbQvwLhAf5tiGAunudpwJpDS1bQj8AdgArAHeAuLDvR2Bf+P0cVTg7LhuD7TdcJo9nnf/f1bjnEkVrhi34LTLV/3PvORT/ndujBuBC8MVY63lO4C24dyOR/KwYS6MMcZUi5TmI2OMMUGwpGCMMaaaJQVjjDHVLCkYY4ypZknBGGNMNUsKxvghIt39jXpZR/lbRaRjEGUm/fDojAkdSwrGNIxbcYawMOaYZknBmMBiRORNd9z799wrkh8VkaXufREmu1emXgWkAW+LyAoRaSYip4jIIhFZKSJLRCTJXWdHEfnMvV/BU2H8bMb4ZUnBmMD6AZPVGbe/EPgZMElVT1HnvgjNgItV9T2cK6pvVNWhgAeYDkxU1SE4Yx+VuOscClwLDAaudcfcN6bJsKRgTGAZqrrQff4vnGFJzhbnbmmrcQYIHOjndf2APaq6FEBVC/X7IbK/VtUCVS3FGf+oW2g/gjFHJqb+IsZErNpjwCjwAs54NRki8jiQ4Od14ue1Vcp8nnuw/0HTxFhNwZjAuorIKPf59cAC9/l+9x4YV/mULQKq+g024PQdnAIgIkk+Q2Qb06TZD9WYwNYDt4jIyzijhr6IMwLrapyRL5f6lH0DeElESoBROP0G/xSRZjj9Cec2XtjGHD0bJdUYY0w1az4yxhhTzZKCMcaYapYUjDHGVLOkYIwxppolBWOMMdUsKRhjjKlmScEYY0y1/w8kfBYnoBIYOQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x154ed6354e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Plot accuracy values\n",
    "plt.plot(valid_acc_values)\n",
    "plt.title('Validation accuracy {:.3f} (mean last 50)'.format(np.mean(valid_acc_values[-50:])))\n",
    "plt.xlabel('batch')\n",
    "plt.ylabel('accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}