Bayesian_CNN_with_MCDO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Loading and normalizing CIFAR-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "train set size: 50000\n",
      "log freq: 50\n",
      "test set size: 10000\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "\n",
    "train_transform = transforms.Compose(\n",
    "    [\n",
    "#   transforms.RandomCrop(32, padding=4),\n",
    "#   transforms.RandomHorizontalFlip(),\n",
    "     transforms.ToTensor(),\n",
    "     transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
    "\n",
    "test_transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=train_transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
    "                                          shuffle=True, num_workers=4)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=test_transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=4)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "print('train set size: {}'.format(len(trainset)))\n",
    "log_freq = len(trainset)//batch_size\n",
    "print('log freq: {}'.format(log_freq))\n",
    "print('test set size: {}'.format(len(testset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1000, 3, 32, 32])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztfXt8VNW59rMyjonTxJiYQkMwJ5im8qEWoSiCHBRvRaxivfToaSmiLbZqtX5Vj1ZtQdtTq/ZYbS1KK2q9t+oRLOjBUigHS5GIIhKDMRAD+SJxJA1Jx6TDzPr+eN+133cykxuES9L1/H6wd9Zes/Zae+/Z87x3Y62Fh4eHh8fAR9b+noCHh4eHR//Av9A9PDw8Bgn8C93Dw8NjkMC/0D08PDwGCfwL3cPDw2OQwL/QPTw8PAYJ/Avdw8PDY5Bgj17oxpipxphNxpj3jTE39dekPDw8PDz6DrO7gUXGmBCA9wCcAWAbgLUALrHWVvXf9Dw8PDw8eouD9uCzJwB431q7GQCMMc8AmA6gyxd6JBKxhx122B6c0sPDw+OfD42NjVFr7ad76rcnL/QSAFvV39sAjO/uA4cddhhmz569B6f08PDw+OfD3LlzP+hNv71uFDXGzDbGVBpjKmOx2N4+nYeHh8c/Lfbkhd4A4Aj193BuS4G1dr61dpy1dlwkEtmD03l4eHh4dIc9eaGvBVBhjBlhjDkYwMUAFvXPtDw8PDw8+ord1qFba3cZY64G8D8AQgAWWGs39nWcuXPn7u4U/qnxwx/+MOXvOXPm7J+JML5xy28AAKvXrQ7aJp98EgBg3k2XpfVf+d6OYL+2rhYAMOvM4/t0zg3bZIx7f/4dAECspQ0A8PivFwbHXvvLWwCAq677WtC2qb4aAHDrFbemjbs4PId2kunnjMdlPxSmbXt7+jHwfjyU/tl4q7SFc1LHSiTkWDKe+jkASPCc2tQYcT5/wTDadqhpxLhfWAnHOWGkwY2b5PO7Nem1QK+v4+8AgDMmSePE+M9TxvzL8/Ld3syuEkVFcjyb55Gbq+bG8wzzsYiaa0EhbS8plrZ8vr5HjB8qY0w9k3aGTuDFLA+O1d35ewDA8qiM8U49bdtapK2VtcNtfB1i6lgbPWJoUfegdRdt01QUAO6cTNuTSqXt0idoW5uhf+fvdl+wJ0ZRWGuXAFiyJ2N4eHh4ePQP9uiF7uHhCGWsmSjMprc2BMdqNtL+N/9dmLFjed/45lVB0+iJJwDonqErMoQ83ja3bFPzYKoWJi2iJqHbok3UPyaMPhFTVLgTeCkIK3btGGOWVlLyvmPXUP2DCWiWn0jv11kIyMlR3d05FTPuYPqdoxh3bj5tnYmqTZYZQLP8ZNqOkgwS6ceQgdEjTid7ddFzQdPEs1K7vPOO7DfytqFJ2gp4m68+EzmEt7yWLHU9xrj+au3OzWJr9fagrWI00+qhFbTNeU/mUUfbNnl0Aha+rU7advIDF+PrHVViTzNv9TOZCbMOpe3XJtJ26RtyLBMz7w/40H8PDw+PQQL/Qvfw8PAYJPAqF49eo4VF0x/dsSBo28oy9Lo31gMAcpPZ8gEW81f/uTJoGlrEgna2cIlX312vu2eU8PMytE0++vOyf83dNLen7kjrV1NHBtDGukZpbMswIGNIUdfHUjQRrDqJO4NmBiNqTMnlJWy326kW2Mg6A2fY1P07uC1bqR1CofQ2BzeGVtvA7WfQMMV1W7xTvwxrSTGKhg1toy0ZOhKKD5X9tp20PVwdz+dHJV8ZRXNZnZLHNzxX3Ysi7nfk0dLWwtcvMupYaRzOxlCM5q3Ev0QKyHDb+EfpXlBC23VqKUn+yKGsD1JPNXJZ/bJZtTkb8lTV8cZvug/Q5litUXwVewWeoXt4eHgMEniG7tEtapWB7epLrwMArFy+MmjrYGtbwlnroKOBia48ueCpoGXM8WMBAFOOnxy0bc8hOuiI6yNL3wqOFeYS58gt3Bm0nTZyEu1ocriD/NA2R9Mdxzoc7e2GlWtkM0vsUIawEFOfpGKpzrUvyWxWuxw6w+em6r8HTWVjiS5HCsUqWu9c4HgtcX35eD+mWJ9zP9QE2rn/OfYOxdAj/NkUl0o3RWWcdQbYYC264z+4v5IsEo4KhjV3TUWumsehfPuKVHdnzM1TDL2A2/KZoUcK5Fg+zy2k3P8Kw0zhI8Olsaicd1zqE2Hv+dy9XBH6KN/HYmWdbeALsIItoNre3bU5HfiPi2W/8m0++xdou35LNx/sJ3iG7uHh4TFI4F/oHh4eHoMEXuXikRE17Kc7c8blQVvl6jUAgJxscQTO5VC+Dif3Q+TnQ1g/0NhQF7R1xEj2LigUWbp8rJJ/AdS1iy5lzl3zAQCnnTwkaDvtNlK5PLv48aCt6j1SA1VrZ2JGrLUjra07fOK666hN3o/roeKduqn+QeRne3PQVrWWjMPTLjgvaNvES3WqBj2Gy2W3U6mWnCEzqebh1CRORZTQqigeVxtKnS+99qkPdp3qRUenHsyH1NwkeDSTCZuQrdQrbnm5Sq3hjKH5qs1Fg7q2mJpjm7uU2cpEnuc6Kl1VC4el5jujqKgIS1gzUzZSujewhu+M06Utdwit67kXaaUrauRYphSDbrjVoi3EkWW0deqs16szfLCf4Rm6h4eHxyDBPzVDP+Uo2R+eQ7/6T6zvKf5r8GLVOom2e3TeIwCA9ZXrgrY4W9bCiqolmLoewgw9FBbGlsVWt452oZP11URh1u0U42XNFmLctxcSC0+OLAyOXfITyrUSXbYibb512ZKQ47cbXqCdDBGgWeG+8ZZM3d2o+lBnzz79t2PNBfkiWdQ9Swk8whcLQw/Oxf2zlPXNGQY1u44590ZFE527omPEmtU6f7r2DEZRvZgQs/DsbiJctZtjMpinNhd2gpq3s51GlAHUketcud2IcFvIrV1syqjiIOREvXxHQ0ezpDdELTDfSXxscd74bHBoJT/ODeprPpojOeNK2IgUko/prTfSBRz9B3nWnIF5vXw10PAhH1NSSR7X8tnExtB1EkS91+AZuoeHh8cggX+he3h4eAwSHAAqFyeydefd2T2uv5BCsGrqxNFzYSWJSFNLxIDyf2eQqPuF0Rwa1iGZgtqipG4YVbI+aPv+EpXB558AOcrA9dr/rgIAxNqiaf1iSjYVgZtk9OykktVZ/ZKjrGPHlJMT8eYacQqv3URhez+5nSI5T/ryacGx0bOn0lCl6cVRVrwi+3V/Jof58PB0FcAnTX0zironMa5SyLqo0IiiQE67ExgItX2Qj5UOOzhoaubww9dXStmAsrJzAQDVdfS3Tm2biW0Fbeoyu6hR51eeVF+ldtfvH2qQDEbfIMkaqzhS/K4zRKUGHbrRuOgFxF0iM22c5X2d2tftt/N8c+rlWB1/Xcf8h7Tdfwl1qBguHUtG3Ek7W8g6XL1BdB0R9mEvUvEVUX7EH3xM2saMp+/+x/wMRMW2jWu/zX7uIUmxVcSvnnZlr23nNbfzNe3IFH3bz/AM3cPDw2OQoEeGboxZAOBLAJqstcdwWyGAZwGUAagD8BVrbXNXY3SHh35CxrdbbpOk7tFdPYdUTRslSR7ufvy7tNMkbHL1AqJv4TahIZEIMaSFzy8FAHwcl5/M1hxi6MWSJx/PfI/8mKq2i8Hlw+10jvmv9rmWxwGPJYuF8lZvYtcv6FysbLFS1DXBVrwws/CwMooOLy5ynYK20hEkHeXlSb/alS8CANo5onPZ88uCY/X/j85/xrniUzbzynsBAK8seEKmxrcy/jHd74aEDQ5t354hn2w3CNz/FLF3DFPLCY5NOyOZ7p/k/rnKSBY66gwAwOt/eDhou+5mYug1zpUwA8sPK2bnCi8kFePO/hS3OYlBF6dgBh0SQUGIeSZDqYsU1cccC8/UvxuEFRt36X61UTRI+6vaArbO1/IUSdeDahaeV6nr/G9P03aWKoaZ/0eSLrOYLWsh4nAujlGh8sGU8f445T3rfAFiPLeIGuSc66gERGNUJMlHXyJGP0oFKgfSHN+zrO4s6v2E3jD0RwFM7dR2E4Bl1toKAMv4bw8PDw+P/YgeGbq1dqUxpqxT83QAp/D+YwBWAPgP7AZm3zQDADCifETQ9uACCiZ54ZXHM34GACJxYeMNL9IvcnubUIjyUmKHhyjGk1dMf6znj255V37PcoYeBwA4ZZK4mRVy9EZpqYxbVjoOAPDlk+mnvjEq7G9oHrUV5EqNrAkXnw0AWLl8TdB28qV3drmu3UWmYIcMZC9j3YI1az4CALzyslJKB4lSulOSAjnsj5bL6fE0Q49xgpJYq/iINRbS8Z07Mgh0uzjyIva5oKnm1SUpWwCoOIrywcy65vyg7dBiso+EOelKScgExx773UMAgBdeuEidjObxwcbl6Iwd7L2pc7lkMdMuVKzzMy42ih+PmF6SuwyKoR8xnPLX1C2bH7TlN64FABTlkx1IF1JwRR5C6uaG+AbqL25cufYBQJYuXOEYt2bt7rimc7wul+Wwdbs61h3ty+qd7cu5YyYyFAHJyZBN0rk0Fqu8LS4PS5FiwU6gXr9V2qp536mz/1W5Jx/Jr5lJU5UoXkoBSN/Kle/owhfp2T1SHkUFcoO85GyJTtq8lhh6mbzG0MKvhtUs7MYPYB36UGuty0X6IeS6enh4eHjsJ+yxUdRaawHYro4bY2YbYyqNMZWxWCYe6eHh4eHRH9hdt8Xtxphia22jMaYYQFNXHa218wHMB4Bhw4Z1+eI/46JJwf6YcccAAOJXiJz40qu/T/2AKqdeX0diX90WcV2K5JAcnF0kiSKKqkkmPqGAEjp86TJJ4XrbnfcDAD7MF/VKyfFlAICmjRISVsR5THK4+kCRshSNKaZjxcXKdJZP/SaU6cqJ/Q/9W+mML/FM2hIlITsD3NYt5H7V2Kh8xIJ1ZZDLFQ5hvUA+J99oaZFEIk3sD9bRIW2RjeSu6FQjGZF8r+tjAGZ9k4oUXPGNC4K2wm4ur6sMP+ui09OOzcmgcnEqgK2qHobLxxFTbmnOmOdEaaVtQsipS1RbU6t7LkRnEHuDfOWOOZVULiuVp6xzOdSqs+Ac3V0+XYOU77dWjDjJXxfkKMih5/Toclp8pXqe2t2HdaZcZ2QNda1H6MhQx1RfI5fLRafZDdIE8flf+ZMakD977Qlq3nzfXxdtCRKcqtdpr5ZsUkPws3C+qkHaEiMFw4NL5SH6xUv0zDoF7PYVokYF6CblFYtjxoUzyKLaXCUuksv+QtsGvt3pjrf9j91l6IsAzOT9mQAW9s90PDw8PDx2F71xW3waZAAtMsZsA/BDAHcC+J0x5nIAHwD4Sn9OqmgEJUGYd9eDQVvsSmJ2r69+GQAwacKZwbG2FhIQjhwu9Ome+54HADz3cfr43xlFlOD+jd8N2n4+idwmV8wTV7iP2ZZXVy8CyAtPEVtv4WPHjhXzweZq+ike8zmx5IysJ4tIVm5J0DZ5CNGElU27H0zVGdVrNatlaqQiTHYxHUvoSgchOl5dRf5goZTfd0elMtQpyxKukeRAojYWERwrB8RAmugQw3EiRCxodEVF0LahqY9p6HhKj/9GIkHq64miHTOFxq2rFxb84Xs074d+eV/Qpm2EnZHvjJ2KobvAnOa/SVMlz+NQVzZNsc9sZpgJzdoDaVGMxK8tIbfd73zlduqjEpusqqOtZrpO+uouPY12scvwKARepGGlKS3NJsZ6OOj8h4bku9Tezv2SYmiWaKqu1aj6yXGFNnQxi4gLLFLM37k3rmB2m608TkezW2G+uqavcb9GqX8Ct+tI+wR1rb7OdvHVEj+ItgYa5Jyz5MI9y+lf3Dd//kvynZ59MhnZMeLloK2knt4fL/1Sxl35Dm2dgHCiHEInQQRA5tKLfUVvvFwu6eLQaV20e3h4eHjsB/hIUQ8PD49BggMgl0vXKDlOxM+XF1FdyofvvwEAkLVF5OFt79UBAEqPFh/y6VeSf/tzd6T7sm/YzgJ3rcpnWX4NAOCUH0ttwvsu+joAoOKz5UHbEUWkOlm4mAS6DevFuFLGWpXciMiQ+Vza/KgSEef++9HvAwAOn5ZeoX53seIlMRon2QSVlUhX6SSVJ3qCLXZ1fD20D7kIgOlqmJyc9DqS7VxgU+dtyebxYm0iWNY0keF17OhRQdvIsvEAgOo6ZdnqBqv+QgULij8tqrDSEtIBfLidDLzRHXLOLdX0rPz0DlF7oXhsl+O7aMxMNUI1Eqw5aeb+zcpg6lLP6KjNsn8dBgDYsETE95Wg67EANO/so+WZX8c26pRMNDyPlKDDDIZPh0iGkp9xVrUMCcnI5RFyGMhPkMpsaLY4VMc5fPOTuMo5m0Xqo/ZulFf5Sr3iDKoFynjtUvC2qQU2sf08l6/fOHWbqvnx+FAFkj/NKtVMa/8a+Vbg2q9KWyHPqUbFDNxwM117rf64gUMc1rE28MafybM5+2v8vT3ua0Fb8VAauEilAl7daT6vZpijRh8DcTPCM3QPDw+PQYIDmqFrhIvIUHrOFMpC8KMnLwyOnTCCWHV+ljCf02aS6n9GBoZ+bCmxj1fmSeL7qfdcw3sTgrZ4GzGSaTd9Xz6cfzgAoOJzZLBdsmhxcGjWzH+ncRe9ELQl2CoWbxMmU1hEfOLlG84CAJx1txhXdhctKvIyzqwpJWMeGy+TmnYy82raTtwkGk3PrJgaW0rjtseEyYfZApc/hKSj4mJx72poUCF9AeiaLl8trqDh3L65dL65gQzNORNFImtmFp6oZePsdmGTLVGifZu2y/oKsojdK1IdwBnmdC6SgJxmokCO2imy2uIMfupyj+TAwusWCle7bDqFMTYup3w2x553vHxgKY+hbmRQD0O1hbJT++k8LK6bNqLm55Nx8/CkcNJ8jgzODpNFODcihu8iTpHYoQdm6aspP1MqRkKeYqtZ/FF9TZ07ZLu6Rjn8uE2fQtu6OrUWd1/EWxBX8zUdnSGic9ZtJPmhUN3lt8gAGuuQtZ9Gwd/4RaV0u4G/wt9ioU5X9Tv9xB8AAP7YrurSjaSFXSq1S3CNDrwGMFntr8TegWfoHh4eHoME/oXu4eHhMUgwYFQuDi0tJOqdcZbkuhw3gsT8ULb6feIkUA9dKNGBW1tIzDrpC2UAgHCrREa2v3wzACCnVNQ2BbnkY7763nlBW7SBP8MhbRElmra2ka98yXAxvm16i8SyeLOIeDm1pIroaCYn20U3nRscO/dOKX7QFzQrdUky8BeX44lMAX0sqjdFSf3Q0qxVLm6+4fQ2NVY8QYPkcVanuPJzj7Z0nba2qU2KA4Ta+pYKqLGBxn3hCVXUET2nyK1a/0CwP3kaGbSmHF+c1s9dKxUIiPq/p3UDyMaJUja0Feh6oHzts0Pi613C9yWUlGfGPW1hDgvVAcV33Ebb/35K2laybS6mLaV8H11tkZTCHNyWH/5I5hGhexrpkHsQ5oxe9U20iJptorKKsZ6kIyT3NiuHVHzxWNfXvUBdvw4eTifncjVKd6ohzuB0uWV8W6rULS7h8e64XRwXwuP5u6b94Ttc8RTetkoxFeTQBW5V/V0dVTHTi/95I8/x1tFy7ELKt4f1V4o6dzSrbSJKN3Mlb3/N29ex9+EZuoeHh8cgwYBj6Dta6Sf+/CuuksZjR2foSb/is679t6AlzC6HSxb8FgCQXyhs/LkFZAUp/5y4KH75YnJbvOeuu4O2KV8g68uhnDykcbv8JK/4E+UFmTxxXNBWy6W+hw4RVrGBK98nEsQSpn9b3J9OWUAMfUWX2XEyozEqbpx5LDUklBHLkeq4yuEZzuvcpmm8Y2g6A4Vj68J42jvouEu8VrtF+ZR1aFNS10gE0kDvyhFGsqmfNmwFCKyGmT6p3Cc3kFlqyvH/ltarkaetvTjzPk3b1o9UR+4XdelzVf82vmz5EYmuzElSApRl1dKxns2yVZwDZ7wafip3mzpT2hayO92DkoEX6zYSxe2I00mTqs5bgkNcK4qEGucn6eFKxOUhO6iAxIww19Ur+pTc4x3t9FktFPTGxS5fPTotLDVoj9cYX7+Kf5G2C88mh4VoIz1Hp02RY89xgOYFp0vCmyOG0r4SHjCGXR03szBdo1IUTeCv5lEqbPNYvuhFyqu1gr02T3J21TOVsTqPFlF3/aqg6cTLeT7SC84mHO+01ShQ+7tVIagTPEP38PDwGCTwL3QPDw+PQYIBp3KZNIMsEueNlzIk408kC8qXzjsnaAux/3ckLOFqZSPJ73vajaRWiW+UDD31rLKYcLaMgYnfBgCc0yTC0JgSGnfOLT8DAOSr0LDRnyenWO1/fc5FNN9iVXqltp7EyYIiMgbuWCcxZSNLSVxe0dRd+qh06FzzsXaSZUPaf9mVWFdGziy2/jn/5XjGuke6LT25FNjAF2ulc8Y/0YJjSoxjl3A+7C1N7rp1r3LpaHPGW1nL5KvIYtYWJRF8nYQYYOxkSuR22QypWNSwReWp7YQmp73SdMepDz6l2tiYF/uAt0qmjgaJr+QarAumq4Vv0gHc/gqr69aJ9bV8rD4ZYTqryaZ/T9qei9IzeN/9dM51W0QvVMAllkqKZTGHtJKqJZklz1heHt2DedeRVbJWVSy67Mq/AgA+VtGjKVnHuoJ6/lyCMZVdGR/z9fvWeWrccq76xQ4LRVNEKTF6Ep3z4RtFvVjFRuKpX5Yhsl00KEd5niqZuTH7at6ZoJRbSTpn4j1xCqhcQ6rSNzbQhE9Urux5Z1Iaq7J7JIx12ruUfvtBKawVPDLdKRLbMrTtCTxD9/Dw8BgkGHAM3eH//kwiNH9122wAwLKXfx60dbBP1BfPFqvKOWymWLOKfKFCLSrVKxc9XF8p4WKtG64HALQpV7ycsfSr7LKc5uaJlSf6ETGHpgaxwhw1khh/e1JFcvJPdnFpGQBgZ4sca472jZk7JFMMoMS8Q0o6CXFUaEJXXGAOkdXuroNmjjkpfQjus9rYSTSrNXBR7HtK4PIKLjhyNUlEt//g+m77J5LErsceI35x55xOjOuIyHQAwLEzxJI4ajQbtJQbnaNGc345J/0E7hboy+GEEh1+6259p+ru9Fk2hiZVaKQj6zE1kSPJN7Fu83cAAD/+pYgWczjyOPdkGUMFXwa4kIe78HY65z0bPh0cW76cJh6KyzOZjNNzGskRPtfO4ZpOBhupPUnj/Hxqhh4kkOm6wEWLSjXsApnbldDmarKOLFemQS6yEvocuyWrAjWIUL/Ln1cRyMspbXRNpTyTFRXEuCdfxr6PoyX6GyH2OUxxx6XxQkXiwjp+ItH7mqfIZXnFH9cGx84Bf1/OlGj1Hyz+BgBgWfg3QdutnOH7AWbtmYpG9Ef+Fg3P0D08PDwGCXpT4OIIAL8FFYK2AOZba+8zxhSCyl+XAagD8BVrbX943vQKkyd9VvaXU52quApQOO/caQCA2x6QqIz/WUpuRrXsx6S55Nmjifu8uvStoG1VDf0Sv/W/90jHIqJlt/6ZwwRaVAmzHcwSGoQtVC0nxh/JEkYwejzp2osKiHGEyyXLw7Ra+o2tfVz06kl2bkokMqT8Y9RERRtXwOXgdPRJSwuxsp0qyGJIIelN85mtRCCMPsbcIecgOWduhMaN7tQ8gPSxbbFMgUiOvWlemR6Isu61PwMANqxjxe1BR8vBXRt5R+Yx93s/AgBcct7ZQdvhRbSWwixSdibi6u46GqQr7MX+kTaPABlMBcEYmcwMjqnrtCZtfH49jzDXbYvommucErCNpMxHXpRImna+f6dEJeTltPNOBgCUd/0o4Ppj9T5dj1fqPx+0VS6nEhAtUVlMfT19dS+bQ89/bqG473ZESARIkb2yupkAo1ER6Tb+ShSqDIzfmUFrwXGqtMJ2fu7b+L5nKV9CF/rTIuduraPvaFZcnrvKpfSsjxvK1zmknUHP6mbGyuWWs19WMGs/XHktgoOwsPyRoOmFe2ke00ZKt1PYNfIqpVff2+gNQ98F4HvW2lGgohtXGWNGAbgJwDJrbQWAZfy3h4eHh8d+Qo8vdGtto7V2He+3AngXQAmA6QBcDbDHAJyXeQQPDw8Pj32BPhlFjTFlAMaAyvUNtdY6/6EPQSqZ/YqwkuwXryI55547fxa03XBz18a2R9anqwJOO5S2VWsl2eWESbN4jy1RYWXlGcGi3QgxJB7JeVtyzpQaqPnbSLSrXkVqm1KlpjhlEsls1e9Jnc2WdtIBhCPKKNUJjR9uVPtddktBewNZ/4oPofFDKSYaUs2075K55XI90HC2qAziXFY+vovWWVAgMmceq350sYnYTmcQ1OItnTf+Ca05p+BkmWMz3dRZM78VtP3gnhvTF9PJCzGkjMu1leR217ytLmjLSnZtzCvm5yimNCNtLheJ7ujUKc5AmKXGdMUwW5WOJp7BpOlyipRO5f7y7Dy9ge5p8UiVypavacnpxwVtXSewFUwtVfszyY9vjXJN/NNSTmu8nr7SK98T18CmuDNaptejTSlW2glR9ZVy3qxjlDqodBKrm7QarpDHK2SXwLBSx7TQMxZ7RQyUNdVsRFW2+1ourxt+nJ6x0dmz5ODJrpatrvDpoL9fbEjNJxXQJ/VyPeq20TljDfKMPfgSn1ONMI5Vd865USXb3WvotVHUGJML4HkA37XW7tTHrLUWUBVnUz832xhTaYyp1L7SHh4eHh79i14xdGNMGPQyf9Ja66o3bDfGFFtrG40xxZAEZSmw1s4HMB8Ahg0blvGlvzdx/U0SgdEdQ8/42TnkilRSriw57jdpI2mbljzyYHBo2mzK/YLjxFUy6fzjwurHrJyYQB4XagiHPwkOVa2mBPxTJknG/u3Mbqrez1SAYvcRZz+6xk+a+O/WDL2EhUSbMx1P7RdSCVDmPUTZDY8YKUa919eS0a+6SvjKTo422fYBGaLO/KJo7w7iygxXzczAqJT3ZLT6bVpDC63lnfVSMqzqrTcBpJbkizhKN7YCndHIzLVEyZylXEsjoQSyOOc96eBAnXiHuj9J2o+qrH5xZ23tUEzeZSt0vqwRxbGaiBFnRcTN8XzFzPcU49X6xs8g4+k03l50hUhQTR08pyw1t3hQOkONmOqEtyslKydtc7QtNRAtVGPYPSt8odf8Njj0i9uJmVerDIyTWZjbKqSCRiulAAAVcUlEQVQdWziJ5Cb+3qyplvs++z5OhjNRV5/4PNLB13k8nb9kvDai85w2vhS0LB5HDhczrhRp46ev0TaH37JlaoS6XRlO2Q/okaEbYwyAhwG8a639L3VoEQDn7DsTmd0sPTw8PDz2EXrD0E8CMAPABmOM8+n7PoA7AfzOGHM5gA8AfGXvTNHDw8PDozfo8YVurV0FwHRx+LQu2g8Y1Nf33AcApo0i+bNGJbFo4PwujY3iuz1r/ETayaH+jdtFyKlfTSqU0hxloQmzGLrlPWmL0fGS8Zxk4miJZCuuIuteVq5Ere1gNcLHUVlMelmGviPM4nLRQeRI3bRLVCrOtVoL0b2Jaos2iey7+BmKenxgwe1B27GlX6CdC77Q5/l2Ru1GuVf5ETI4NjdsBgDE46IbOWokGd+GFMpVy8km1cLWFnVfHNjA2rBd8qo0sD0zrIzgBXybi7iaRX5EohoLktTmVEAAsNM9CwkxY7Zsp3uwnVUMO3UUaR4ZmF+vEgN5TZR87yt01Csj6sIgNqgcv7k0bjxP9FNl5eTnn2EIuETUt35bDISX3chpYiNDVE9ea7xr43Ku0lQewpqZE8Yqw3DYqY9UdPQOMkI2Lab4kScWyKHXWNWi65Lex4G1YrKUgNwIq1xq1bEfn0Q953xbUlzP+sWTtBM6QfVUOX0BAAerffbRz5PvS5j3P6OCXp91UTmsXpmusnyH+JbW9i7dUa/hI0U9PDw8BgkGbC6XTGhRHlQucd/ZZ56a1s8lTjvlGGFsjy74FQDg6Rcll8aSP1IUaHWlMMHKNWRsO3sKuShGSmWMzZxuL7bqtaBt5Hln0E6RRD8+cikZZ3OYalzya8nwOHoKGQSffkzm8cLLFEmpfYT6ytDHjaNSfBdeLPknzp9K56p+hCLeVt47NzgWTxJr0pZut1+n2jT76YyHHrkDQCpD7xcwodu0QaxjBVxrrYPT+g0ZJSn2wiGSN3ILhD4VOLfFtRkYujtBm7K6NpOUFs8WltrEhSSa4o6p6TvE1GuVMr6NI8o64V/FLTM8gmhb+VBirrkRSeuX10rPXUFUooZ/cS8ZzXOHy1oKS2nNJbksSbbJPBrZ/bUpKjmKiofSZ08984Kg7dhilZIQwNeV7XXhF48EALzwsnK8c9JGu15zKj8sUBkKnd3zhAod+emuvQopXf8iAMClVMpXD/ooFo43SDB38MleCuJBvyvmibz543mkLZ4gCVzxlQto8pNPpwjehMqL9GHt0wCAhY/Ls/P0Mp5bN+deKMldUcKXqmuT8u7BM3QPDw+PQQL/Qvfw8PAYJBiwKpdVfxE1yH3zqMDic0+o8uioRldwAvJL74gp5bMnUIZ8nTyoNkP2+dItJI7HTqZRvjRTfKbzjmP1Su0K+UCRE9GPCZri/+A6km/THC/ZJuJwIk7+y1UfiPA2bgqpa1av63pNY0eJGH/qFEr2dcMttwZtQ4oPTvuMQ8VFpIYp+OuLQVtdHYnXm1RFgggnnNKmsSN5+7Hro445tlCz+C9yrrMndjmP3qK9iu7Bkfmidkgm6WblFlJbMixpjds5Z2ueqkGaE+/GETjC/XSq4QSbEDtUOleXIjfOx2LKn9r5uedLkquKoXQtzxh7ZNB2+30uGRU3ZCtTZdVS2hZL6O+FZ14CANhaLw9n1hZaazF/NFfFPJRwcql4WEJFK9fQORtrZdzrv0th0UWF6T7Zz39vGABg2BuicmlsYp/7uE73rO8+UKKWMpwvZeFY7UdBc1v/S6nZu4SXXM26vBal0Xmzjra9Va90B63ecGrD2k3S9sR/8lviP3+GvYGGrm3JewTP0D08PDwGCQYsQ29qlqi8FUtd8GrXDLYnOA+j5h5qQt111zUAgGMvdkxDsxJOVFGuEmds5PLsbU8HTVu5GMSok8l1aoeKamzj8ugRlb40xgU2cnIzOZoR3ti4ovuJd4eRlIp40o8fDpomxfiKNAkfqq0ml8SmFsn80LSdfPzaE0SlmhWjijJjLSnuDydLQbiaGObIos9I42iKJI3WUYRj/QaxQMU5qUiLcmXsaGUGlkoquY15TkoKXm5LKEOpc9lzRtGQ8kFzUZUFwuhDCZIqS3N1bTYeL8Zhm5rlJ9jFL18khQnnUiTl+t+/GbTVPMTSXJylkoiKWC2iORWPFLkq0kbRsREVeZyYR7mP7r4lU9QkYfLxYtB89nl3/q5NeaPKZe1Dk7TO+mVi7H96EUnIT78sn2njR8s9RppxKtOpRxfwDN3Dw8NjkMC/0D08PDwGCQacyqWObaG//rXU7os2vdVF7/7BKRUqfelQJ7o6v11JrYsY+xxHxL+3vZlE3jfXiJGzhY1zW5tJ5Fy4dFlwrCNGRr0HnpCISydqag/ekaf3cRHdIZCMxZCIBg6X3Cae5uXtJF6XqySyMU7GVdNO23XK2FNSTLVCI2O7Tvu7OwhVHEE7w1V2Kd4t4hTD2RFZy9baOgBA/TZRHwWaoUR6BtBw4WEAgHiHSkbmytbrBGasRkCc9XQpKXn5q5UUdUn1Bup39c9E7YA4X/wstsYnD5Vj+Zw4LCbGyNooPQ0Nf1Oqn8BQy0biDmXM5QxVjZvUOp3qTiUC29BGaql43iIAwN3XnBscc6Pdf51UCXvhZTLkx5Ndq1zylDqmkXNWPfETcUR49R3aXqhKfjqV3Rp28d4qeeuCx1RXtPVIhWfoHh4eHoMEA46h19fTT311zeZ9ds6dMWEaP7mHoipvZVY25Eypa4mI83n8IGhKcvrUxqjwilNOp8+8s5GMezfdL8UpMuUgnj2BjIrb23uu47hHWCqG23gt5aXJigjb64jTGmLbhWW1MHPNShA3mKS8+somShrhPYYmgq4m5xZVoWEHSw15dCxPVXAflU2TGjVK0vgij6Sutx/6z7RTORtkR574sLY6I7Guch/m+5HNEpx24etwx45QayCpoX2HGsOl9GXDdyrL54u5TSSiX93ODgA5Ku1vLvdL8n0J6/FZrourBCjtbAyNKYvwDpI87mskKXPzNnkSf3kXpZHWaUeSTsZpVxS6Ex54RqSCFd18Xder8MoxXB9lnBPqVJKWun1WsXjgwjN0Dw8Pj0GCAcfQJx9PrlxPPSl6yAc5sOi3869N6z93lgTcvLqUcqKs6qP/07oGoYc7oqRTjoSJKd0YFlZWOITdFfMkiUUTs9nVa4WGjD2ZXBI3rCNFYcbKIAp33H4bAGDW9XsnyCHA8aLMDOfxb71i4+Fa0kFHYqJHLhhKevJQEetld6hacK4wwhbFXEf0pmBaD3CFNnIVw2xh/Xj1VtoWqrR3MZ5H9GNpKyvrcvgE3+7sbEkyms35T6LtigM5oSvJTFS7OTo+G1KiRZzn26LanNtiNo+RpVwOQy7ASYk9jcz8c5SevN0VUeFxtTST5HnsUo3BEpTNxB2vo81L88SG89JiLnaRr+bRwvNIptsgHLpj5Rqrlavwao6x47Q3GKtK1tVJShuPLuAZuoeHh8cggX+he3h4eAwS9KhyMcbkgHzzsrn/c9baHxpjRgB4BsDhAN4AMMNa+4+uR+pfTDhORPcJD1H05mO8BYCmteR+tWzp4qDtMzPLAABZiykyc+X6vkeW1rEE+9wrpC5paRMVw+SxlHP0EpcyF0DZ6RRRWlEpovRPfvxYylg6y+hlU8kq9PUZ5wdty1gOXbJeXAiPPw/9jwIV0ZnLUYqqiGY4QpGt4bCuLemOh1M2AIBoHW1rlKw8oo+GUqfFWCuGY3ANT5SrigGtfGJOaZuS1jXuDJpSJ1O7E6bBBYDqNla/FKgiFjuzkjxFZ9jUeU2cCkXNwxkrk+r6ZfN8A9WFNj3yDHRFhwSrbZrV7A5i9YdzgUyZOc8tK0NbigHWzYmNqDpiusrNSVkos/j6JZVhGir/LIBxh8h+Zde204yo5PNvUo9OxeG0rfk4vb8HoTcMvQPAqdba0aDKqVONMScC+CmAe621nwVFzl++96bp4eHh4dETelOCzkJ+r8P8zwI4FcC/c/tjAOYAmNf/U9w97GCmUTHl20FbdpKYyaRzKQvhqPESKFHLZD3eLiyrqIiY0etrJHBp5leJfddyhffst4SBtUQpq+ARRcLiRsYos15TVCjPBiY8f32YCj+Mn6jc6RyjU3lHLrsm3bVur+BzI2XfGXublQXZseU8ZRyrJ/dGuOr2YRX+NHIsbceJYToYQ5NDd80dE21WTLeWI0zqVSo8xzbbJYMlksxSWzOEnThjYUTRzpauc/btaiejayKFcfP4ysiZncOLGUrzjqnydGhhBqtthi4IJxONcqxd3ffAlTHF2Mr9skPp/YJrqvpnOalAj5Ep1V+40zHdP7tTHyhm3nUNtb6y8kxQYVw4xLst9ohe6dCNMSEuEN0E4FVQxsm/WWtdDtJtSA1k1J+dbYypNMZUxmJdW8Q9PDw8PPYMvXqhW2sT1trjAAwHcAKAkT18RH92vrV2nLV2XCSSKbWdh4eHh0d/oE9+6NbavxljlgOYAOAwY8xBzNKH4wDLbhkF+YRPmvjpXvUvD36i0v2kpw2XAosfXUAV1effSXVBJ1TIb2IbGwEjIRGHGzm8rfJtMShdf/6ZAIDxl5F/ObYsCo6119Nl3OqSXwBYv6+MQMU6dSv5lyNPidnxDNGBLpK0mCMXs9T1G8lGy0wBrlXvyr7zJy/ksfKVGF/MxtmIUkvtYNlbV5x3zuPuXNpn2hkS48owOMSl3lV+84yD4tR2UEIbEkP8v1ZFpOYxyc6V+XSwliJFKHX+5zHth95pKP2BQOWjrkegEVHzSMunov5Opu1A1Cn6xmR36qfHzOp0DBAt7L6Tupv2UlGIwYQeGbox5tPGmMN4/xAAZwB4F8ByAK7i8EwAC/fWJD08PDw8ekZvGHoxgMeMMSHQD8DvrLV/MMZUAXjGGPMjAG8CeLi7QfY1Jh3fO2a+u5g6haIqC+J1QVtVE4XGxcNSiKJqHWXKq6wSJnj9jT8EALQufwgA8Otn5LewfjsZm6aedU7QNv5IkhDWbN67WSVT4DIZ6oyGGdHJDbFRubGFM6jYHDks/z/S1p0mLqjzMVzaWi1tkxLJiTi3uQjR4k/JMWcnHaoKj5Sys+jSl9JOmbODijc01khVd4SdpKIjLt2+i5pUrNmxex1JmQinDYEOpp3OMBxTbDyI7kyxrKbNN93SnMFFMQWun3bddDchk9U6E+9r7LT1OBDQGy+XtwGMydC+GaRP9/Dw8PA4AOAjRT08PDwGCQZccq79jgQFw4ZyyQjY1Cji8LW3PQ8A+MwIVQ90B4nQU86ZGbQVjSAD38JFTwEAGlslQvPrV88FABSXSg3Iv14xBwAw85t3qYkcoC6gxT2oaJwdbk8cnvJMhkbXxqoWfXmcfTRfxeRqF/NOuPR0ih1Y2CYJ1XKCPFZKJRJy6on0Ig9ZTuWiDLfxEO3Hs2TxyVAWt5H6Ix5PN8rHlR96VqAmEYNmklU9CU7tm9Qu6mmjqTkqPpd04yakRUDHOkKioklkkdFelQ31OADgGbqHh4fHIIGhQNB9g2HDhtnZs2fvs/N5eHh4DAbMnTv3DWvtuJ76eYbu4eHhMUjgX+geHh4egwT+he7h4eExSOBf6B4eHh6DBPvUKGqM+QjA3wFEe+p7gKMIA3sNA33+wMBfw0CfPzDw1zCQ5v8v1toew9/36QsdAIwxlb2x1h7IGOhrGOjzBwb+Ggb6/IGBv4aBPv9M8CoXDw8Pj0EC/0L38PDwGCTYHy/0+fvhnP2Ngb6GgT5/YOCvYaDPHxj4axjo80/DPtehe3h4eHjsHXiVi4eHh8cgwT59oRtjphpjNhlj3jfG3LQvz707MMYcYYxZboypMsZsNMZcy+2FxphXjTE1vC3Y33PtDlzk+01jzB/47xHGmDV8H541xhy8v+fYHYwxhxljnjPGVBtj3jXGTBiA9+A6fobeMcY8bYzJOZDvgzFmgTGmyRjzjmrLeM0N4X5ex9vGmLH7b+aCLtZwNz9Hbxtj/ttVY+NjN/MaNhljvrh/Zr1n2GcvdK549ACAswCMAnCJMWZU95/a79gF4HvW2lEATgRwFc/5JgDLrLUVAJbx3wcyrgWVDXT4KYB7rbWfBdAM4PL9Mqve4z4Ar1hrRwIYDVrLgLkHxpgSANcAGGetPQaU+/ZiHNj34VEAUzu1dXXNzwJQwf9mA5i3j+bYEx5F+hpeBXCMtfbzAN4DcDMA8Pf6YgBH82d+xe+sAYV9ydBPAPC+tXaztfYfAJ4BMH0fnr/PsNY2WmvX8X4r6EVSApr3Y9ztMQDn7Z8Z9gxjzHAAZwP4Df9tAJwK4DnucqDPPx/AZHCJQ2vtP6y1f8MAugeMgwAcYow5CJQNvhEH8H2w1q4EsKNTc1fXfDqA31rCX0EF5Iuxn5FpDdbapVzYHgD+CqltOB3AM9baDmvtFgDvYwBWZNuXL/QSAFvV39u4bUDAGFMGKsW3BsBQa60rpvghgJ4Kb+5P/BzAjZCKBYcD+Jt6qA/0+zACwEcAHmG10W+MMZ/CALoH1toGAPcAqAe9yFsAvIGBdR+Arq/5QP1uXwbgZd4fqGtIgTeK9gLGmFwAzwP4rrV2pz5myU3ogHQVMsZ8CUCTtfaN/T2XPcBBAMYCmGetHQNKHZGiXjmQ7wEAsK55OujHaRiorFJnVcCAwoF+zXuCMeYWkEr1yf09l/7EvnyhNwA4Qv09nNsOaBhjwqCX+ZPW2he4ebsTKXnbtL/m1wNOAnCuMaYOpOI6FaSPPoxFf+DAvw/bAGyz1q7hv58DveAHyj0AgNMBbLHWfmStjQN4AXRvBtJ9ALq+5gPqu22MuRTAlwB81Yrf9oBaQ1fYly/0tQAq2LJ/MMgAsWgfnr/PYH3zwwDetdb+lzq0CIArEjoTwMJ9PbfewFp7s7V2uLW2DHS9/2St/SqA5QAu5G4H7PwBwFr7IYCtxpijuOk0AFUYIPeAUQ/gRGNMhJ8pt4YBcx8YXV3zRQC+zt4uJwJoUaqZAwrGmKkgFeS51lpdeXYRgIuNMdnGmBEgA+/r+2OOewRr7T77B2AayLJcC+CWfXnu3ZzvJJBY+TaAt/jfNJAeehmAGgB/BFC4v+fai7WcAuAPvH8k6GF9H8DvAWTv7/n1MPfjAFTyfXgRQMFAuwcA5gKoBvAOgMcBZB/I9wHA0yB9fxwkJV3e1TUHVeh+gL/XG0DePAfqGt4H6crd9/lB1f8WXsMmAGft7/nvzj8fKerh4eExSOCNoh4eHh6DBP6F7uHh4TFI4F/oHh4eHoME/oXu4eHhMUjgX+geHh4egwT+he7h4eExSOBf6B4eHh6DBP6F7uHh4TFI8P8B9toKOrxxE1AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " frog   cat plane   cat\n"
     ]
    }
   ],
   "source": [
    "# get some random training images\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()\n",
    "print(images.shape)\n",
    "\n",
    "# show images\n",
    "imshow(torchvision.utils.make_grid(images[:4]))\n",
    "# print labels\n",
    "print(' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Define Lenet with/without dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lenet without dropout \n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.conv1(x))\n",
    "        x = self.pool(self.conv2(x))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "# Lenet with dropout\n",
    "class Net_DO(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net_DO, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        self.dropout = nn.Dropout(p=0.3)\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.conv1(x))\n",
    "        x = self.pool(self.conv2(x))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(self.dropout(x)))\n",
    "        x = self.fc3(self.dropout(x))\n",
    "        return x\n",
    "    \n",
    "# Lenet with MCDO\n",
    "class Net_MCDO(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net_MCDO, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        self.dropout = nn.Dropout(p=0.3)\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.dropout(self.conv1(x)))\n",
    "        x = self.pool(self.dropout(self.conv2(x)))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(self.dropout(x)))\n",
    "        x = F.softmax(self.fc3(self.dropout(x)),dim=1)\n",
    "        return x\n",
    "    \n",
    "def update_target(target, original, update_rate):\n",
    "        for target_param, param in zip(target.parameters(), original.parameters()):\n",
    "            target_param.data.copy_((1.0 - update_rate) * target_param.data + update_rate*param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Define a Loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "CE = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, net, net_test, optimizer, log_freq=log_freq):\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs\n",
    "        inputs, labels = data\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = CE(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        update_target(net_test, net, 0.001)\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if (i+1) % log_freq == 0:    # print every 2000 mini-batches\n",
    "            print('[Epoch : %d, Iter: %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / log_freq))\n",
    "    return running_loss / log_freq\n",
    "\n",
    "def test(net, is_MCDO=False):\n",
    "    print('Start test')\n",
    "    class_correct = list(0. for i in range(10))\n",
    "    class_total = list(0. for i in range(10))\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            output = 0\n",
    "            if is_MCDO:\n",
    "                for i in range(10):\n",
    "                    output += net(inputs)/10.\n",
    "                output = torch.log(output)\n",
    "            else:\n",
    "                output = net(inputs)\n",
    "            _, predicted = torch.max(output, 1)\n",
    "            c = (predicted == labels).squeeze()\n",
    "            for i in range(len(labels)):\n",
    "                label = labels[i]\n",
    "                class_correct[label] += c[i].item()\n",
    "                class_total[label] += 1\n",
    "\n",
    "\n",
    "    for i in range(10):\n",
    "        print('Accuracy of %5s : %.2f %%' % (\n",
    "            classes[i], 100 * class_correct[i] / class_total[i]))\n",
    "\n",
    "    test_score = np.mean([100 * class_correct[i] / class_total[i] for i in range(10)])\n",
    "    print(test_score)\n",
    "    return test_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "\n",
    "lenets = [Net, Net_DO, Net_MCDO]\n",
    "epoch_num = 200\n",
    "test_freq = 10\n",
    "losses = list()\n",
    "net_scores = list()\n",
    "test_scores = list()\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "is_train = True\n",
    "\n",
    "# SAVE\n",
    "def save(name, net, net_test):\n",
    "    net_path = './model/'+name+'.pkl'\n",
    "    net_test_path = './model/'+name+'_test.pkl'\n",
    "    net = net.cpu()\n",
    "    torch.save(net.state_dict(), net_path)\n",
    "    net_test = net_test.cpu()\n",
    "    torch.save(net_test.state_dict(), net_test_path)\n",
    "    # Place it to GPU back\n",
    "    net.to(device)\n",
    "    net_test.to(device)\n",
    "    return net, net_test\n",
    "    \n",
    "def load(name, net, net_test):\n",
    "    net_path = './model/'+name+'.pkl'\n",
    "    net_test_path = './model/'+name+'_test.pkl'\n",
    "    # LOAD\n",
    "    net.load_state_dict(torch.load(net_path))\n",
    "    net_test.load_state_dict(torch.load(net_test_path))\n",
    "    # Place it to GPU\n",
    "    net.to(device)\n",
    "    net_test.to(device)\n",
    "    return net, net_test\n",
    "\n",
    "def main():\n",
    "    for lenet in lenets:\n",
    "        print(lenet.__name__)\n",
    "        net = lenet()\n",
    "        net_test = lenet()\n",
    "        if torch.cuda.device_count() > 1:\n",
    "            print(\"Let's use\",torch.cuda.device_count(),\"GPUs!\")\n",
    "            net = nn.DataParallel(net)\n",
    "            net_test = nn.DataParallel(net_test)\n",
    "        net.to(device)\n",
    "        net_test.to(device)\n",
    "        net_test.load_state_dict(net.state_dict())\n",
    "        \n",
    "        optimizer = optim.Adam(net.parameters(), lr=5e-4, weight_decay=0.0005, amsgrad=True)\n",
    "        scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.75)\n",
    "        \n",
    "        for i in tqdm_notebook(range(epoch_num)):\n",
    "            scheduler.step()\n",
    "            if lenet.__name__ == 'Net_DO':\n",
    "                net.train()\n",
    "                net_test.train()\n",
    "            loss_avg = train(epoch=i, net=net, net_test=net_test, optimizer=optimizer)\n",
    "            losses.append(loss_avg)\n",
    "            if (i+1) % test_freq == 0:\n",
    "                if lenet.__name__ == 'Net_DO':\n",
    "                    print('NET_DO TEST')\n",
    "                    net.eval()\n",
    "                    net_test.eval()\n",
    "                if lenet.__name__ == 'Net_MCDO':\n",
    "                    print('NET_MCDO TEST')\n",
    "                    print('Train net test')\n",
    "                    net_score = test(net,is_MCDO=True)\n",
    "                    net_scores.append(net_score)\n",
    "                    print('Test net test')\n",
    "                    test_score = test(net_test,is_MCDO=True)\n",
    "                    test_scores.append(test_score)\n",
    "                else:\n",
    "                    print('Train net test')\n",
    "                    net_score = test(net)\n",
    "                    net_scores.append(net_score)\n",
    "                    print('Test net test')\n",
    "                    test_score = test(net_test)\n",
    "                    test_scores.append(test_score)\n",
    "                \n",
    "        save(lenet.__name__, net, net_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6694caca35fb49609f2c65acdc845ab8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=200), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 1.569\n",
      "[Epoch : 2, Iter:    50] loss: 1.061\n",
      "[Epoch : 3, Iter:    50] loss: 0.822\n",
      "[Epoch : 4, Iter:    50] loss: 0.656\n",
      "[Epoch : 5, Iter:    50] loss: 0.474\n",
      "[Epoch : 6, Iter:    50] loss: 0.319\n",
      "[Epoch : 7, Iter:    50] loss: 0.207\n",
      "[Epoch : 8, Iter:    50] loss: 0.146\n",
      "[Epoch : 9, Iter:    50] loss: 0.117\n",
      "[Epoch : 10, Iter:    50] loss: 0.084\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.70 %\n",
      "Accuracy of   car : 79.20 %\n",
      "Accuracy of  bird : 61.10 %\n",
      "Accuracy of   cat : 48.10 %\n",
      "Accuracy of  deer : 66.70 %\n",
      "Accuracy of   dog : 61.40 %\n",
      "Accuracy of  frog : 78.80 %\n",
      "Accuracy of horse : 73.50 %\n",
      "Accuracy of  ship : 77.00 %\n",
      "Accuracy of truck : 77.70 %\n",
      "70.22\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 71.70 %\n",
      "Accuracy of   car : 85.70 %\n",
      "Accuracy of  bird : 30.20 %\n",
      "Accuracy of   cat : 47.50 %\n",
      "Accuracy of  deer : 41.70 %\n",
      "Accuracy of   dog : 38.90 %\n",
      "Accuracy of  frog : 47.50 %\n",
      "Accuracy of horse : 58.30 %\n",
      "Accuracy of  ship : 59.10 %\n",
      "Accuracy of truck : 65.20 %\n",
      "54.580000000000005\n",
      "[Epoch : 11, Iter:    50] loss: 0.042\n",
      "[Epoch : 12, Iter:    50] loss: 0.019\n",
      "[Epoch : 13, Iter:    50] loss: 0.010\n",
      "[Epoch : 14, Iter:    50] loss: 0.007\n",
      "[Epoch : 15, Iter:    50] loss: 0.006\n",
      "[Epoch : 16, Iter:    50] loss: 0.006\n",
      "[Epoch : 17, Iter:    50] loss: 0.007\n",
      "[Epoch : 18, Iter:    50] loss: 0.007\n",
      "[Epoch : 19, Iter:    50] loss: 0.008\n",
      "[Epoch : 20, Iter:    50] loss: 0.009\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 82.30 %\n",
      "Accuracy of   car : 79.90 %\n",
      "Accuracy of  bird : 56.80 %\n",
      "Accuracy of   cat : 54.40 %\n",
      "Accuracy of  deer : 69.60 %\n",
      "Accuracy of   dog : 62.30 %\n",
      "Accuracy of  frog : 80.00 %\n",
      "Accuracy of horse : 77.80 %\n",
      "Accuracy of  ship : 80.60 %\n",
      "Accuracy of truck : 78.40 %\n",
      "72.21\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 73.70 %\n",
      "Accuracy of   car : 89.40 %\n",
      "Accuracy of  bird : 43.50 %\n",
      "Accuracy of   cat : 47.00 %\n",
      "Accuracy of  deer : 56.30 %\n",
      "Accuracy of   dog : 54.30 %\n",
      "Accuracy of  frog : 79.30 %\n",
      "Accuracy of horse : 77.10 %\n",
      "Accuracy of  ship : 77.40 %\n",
      "Accuracy of truck : 77.40 %\n",
      "67.53999999999999\n",
      "[Epoch : 21, Iter:    50] loss: 0.013\n",
      "[Epoch : 22, Iter:    50] loss: 0.075\n",
      "[Epoch : 23, Iter:    50] loss: 0.295\n",
      "[Epoch : 24, Iter:    50] loss: 0.117\n",
      "[Epoch : 25, Iter:    50] loss: 0.038\n",
      "[Epoch : 26, Iter:    50] loss: 0.014\n",
      "[Epoch : 27, Iter:    50] loss: 0.005\n",
      "[Epoch : 28, Iter:    50] loss: 0.004\n",
      "[Epoch : 29, Iter:    50] loss: 0.004\n",
      "[Epoch : 30, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.40 %\n",
      "Accuracy of   car : 82.30 %\n",
      "Accuracy of  bird : 62.50 %\n",
      "Accuracy of   cat : 53.80 %\n",
      "Accuracy of  deer : 69.30 %\n",
      "Accuracy of   dog : 62.40 %\n",
      "Accuracy of  frog : 81.60 %\n",
      "Accuracy of horse : 78.10 %\n",
      "Accuracy of  ship : 82.90 %\n",
      "Accuracy of truck : 80.80 %\n",
      "73.21\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 77.10 %\n",
      "Accuracy of   car : 87.70 %\n",
      "Accuracy of  bird : 50.90 %\n",
      "Accuracy of   cat : 50.10 %\n",
      "Accuracy of  deer : 64.20 %\n",
      "Accuracy of   dog : 59.60 %\n",
      "Accuracy of  frog : 83.80 %\n",
      "Accuracy of horse : 79.80 %\n",
      "Accuracy of  ship : 81.20 %\n",
      "Accuracy of truck : 81.00 %\n",
      "71.54\n",
      "[Epoch : 31, Iter:    50] loss: 0.004\n",
      "[Epoch : 32, Iter:    50] loss: 0.004\n",
      "[Epoch : 33, Iter:    50] loss: 0.004\n",
      "[Epoch : 34, Iter:    50] loss: 0.005\n",
      "[Epoch : 35, Iter:    50] loss: 0.005\n",
      "[Epoch : 36, Iter:    50] loss: 0.005\n",
      "[Epoch : 37, Iter:    50] loss: 0.005\n",
      "[Epoch : 38, Iter:    50] loss: 0.005\n",
      "[Epoch : 39, Iter:    50] loss: 0.005\n",
      "[Epoch : 40, Iter:    50] loss: 0.005\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.40 %\n",
      "Accuracy of   car : 81.60 %\n",
      "Accuracy of  bird : 63.10 %\n",
      "Accuracy of   cat : 52.00 %\n",
      "Accuracy of  deer : 69.30 %\n",
      "Accuracy of   dog : 62.60 %\n",
      "Accuracy of  frog : 80.40 %\n",
      "Accuracy of horse : 78.10 %\n",
      "Accuracy of  ship : 82.90 %\n",
      "Accuracy of truck : 81.60 %\n",
      "73.0\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 77.50 %\n",
      "Accuracy of   car : 87.00 %\n",
      "Accuracy of  bird : 56.20 %\n",
      "Accuracy of   cat : 50.90 %\n",
      "Accuracy of  deer : 67.00 %\n",
      "Accuracy of   dog : 61.70 %\n",
      "Accuracy of  frog : 83.80 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 81.70 %\n",
      "Accuracy of truck : 81.60 %\n",
      "72.81\n",
      "[Epoch : 41, Iter:    50] loss: 0.005\n",
      "[Epoch : 42, Iter:    50] loss: 0.005\n",
      "[Epoch : 43, Iter:    50] loss: 0.005\n",
      "[Epoch : 44, Iter:    50] loss: 0.005\n",
      "[Epoch : 45, Iter:    50] loss: 0.005\n",
      "[Epoch : 46, Iter:    50] loss: 0.005\n",
      "[Epoch : 47, Iter:    50] loss: 0.005\n",
      "[Epoch : 48, Iter:    50] loss: 0.005\n",
      "[Epoch : 49, Iter:    50] loss: 0.005\n",
      "[Epoch : 50, Iter:    50] loss: 0.005\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 80.10 %\n",
      "Accuracy of   car : 82.20 %\n",
      "Accuracy of  bird : 61.90 %\n",
      "Accuracy of   cat : 52.00 %\n",
      "Accuracy of  deer : 68.10 %\n",
      "Accuracy of   dog : 62.90 %\n",
      "Accuracy of  frog : 81.00 %\n",
      "Accuracy of horse : 76.60 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 79.80 %\n",
      "72.68\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.50 %\n",
      "Accuracy of   car : 85.00 %\n",
      "Accuracy of  bird : 58.30 %\n",
      "Accuracy of   cat : 52.40 %\n",
      "Accuracy of  deer : 68.20 %\n",
      "Accuracy of   dog : 63.00 %\n",
      "Accuracy of  frog : 83.40 %\n",
      "Accuracy of horse : 79.70 %\n",
      "Accuracy of  ship : 81.80 %\n",
      "Accuracy of truck : 81.50 %\n",
      "73.17999999999999\n",
      "[Epoch : 51, Iter:    50] loss: 0.005\n",
      "[Epoch : 52, Iter:    50] loss: 0.005\n",
      "[Epoch : 53, Iter:    50] loss: 0.005\n",
      "[Epoch : 54, Iter:    50] loss: 0.005\n",
      "[Epoch : 55, Iter:    50] loss: 0.005\n",
      "[Epoch : 56, Iter:    50] loss: 0.005\n",
      "[Epoch : 57, Iter:    50] loss: 0.005\n",
      "[Epoch : 58, Iter:    50] loss: 0.005\n",
      "[Epoch : 59, Iter:    50] loss: 0.005\n",
      "[Epoch : 60, Iter:    50] loss: 0.005\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.70 %\n",
      "Accuracy of   car : 82.20 %\n",
      "Accuracy of  bird : 62.00 %\n",
      "Accuracy of   cat : 52.30 %\n",
      "Accuracy of  deer : 67.40 %\n",
      "Accuracy of   dog : 62.50 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 76.80 %\n",
      "Accuracy of  ship : 81.50 %\n",
      "Accuracy of truck : 79.80 %\n",
      "72.6\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 83.70 %\n",
      "Accuracy of  bird : 59.20 %\n",
      "Accuracy of   cat : 53.20 %\n",
      "Accuracy of  deer : 67.70 %\n",
      "Accuracy of   dog : 63.10 %\n",
      "Accuracy of  frog : 82.90 %\n",
      "Accuracy of horse : 79.30 %\n",
      "Accuracy of  ship : 82.00 %\n",
      "Accuracy of truck : 81.00 %\n",
      "73.12\n",
      "[Epoch : 61, Iter:    50] loss: 0.005\n",
      "[Epoch : 62, Iter:    50] loss: 0.005\n",
      "[Epoch : 63, Iter:    50] loss: 0.005\n",
      "[Epoch : 64, Iter:    50] loss: 0.005\n",
      "[Epoch : 65, Iter:    50] loss: 0.004\n",
      "[Epoch : 66, Iter:    50] loss: 0.004\n",
      "[Epoch : 67, Iter:    50] loss: 0.004\n",
      "[Epoch : 68, Iter:    50] loss: 0.004\n",
      "[Epoch : 69, Iter:    50] loss: 0.004\n",
      "[Epoch : 70, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 77.80 %\n",
      "Accuracy of   car : 82.10 %\n",
      "Accuracy of  bird : 61.40 %\n",
      "Accuracy of   cat : 53.40 %\n",
      "Accuracy of  deer : 69.00 %\n",
      "Accuracy of   dog : 61.30 %\n",
      "Accuracy of  frog : 81.90 %\n",
      "Accuracy of horse : 77.00 %\n",
      "Accuracy of  ship : 82.50 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.68\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 83.00 %\n",
      "Accuracy of  bird : 59.50 %\n",
      "Accuracy of   cat : 53.10 %\n",
      "Accuracy of  deer : 67.60 %\n",
      "Accuracy of   dog : 62.70 %\n",
      "Accuracy of  frog : 82.40 %\n",
      "Accuracy of horse : 79.00 %\n",
      "Accuracy of  ship : 81.80 %\n",
      "Accuracy of truck : 81.00 %\n",
      "72.92999999999999\n",
      "[Epoch : 71, Iter:    50] loss: 0.004\n",
      "[Epoch : 72, Iter:    50] loss: 0.004\n",
      "[Epoch : 73, Iter:    50] loss: 0.004\n",
      "[Epoch : 74, Iter:    50] loss: 0.004\n",
      "[Epoch : 75, Iter:    50] loss: 0.004\n",
      "[Epoch : 76, Iter:    50] loss: 0.004\n",
      "[Epoch : 77, Iter:    50] loss: 0.004\n",
      "[Epoch : 78, Iter:    50] loss: 0.004\n",
      "[Epoch : 79, Iter:    50] loss: 0.004\n",
      "[Epoch : 80, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.50 %\n",
      "Accuracy of   car : 82.00 %\n",
      "Accuracy of  bird : 59.70 %\n",
      "Accuracy of   cat : 51.40 %\n",
      "Accuracy of  deer : 70.80 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.20 %\n",
      "Accuracy of horse : 77.60 %\n",
      "Accuracy of  ship : 82.00 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.42999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.70 %\n",
      "Accuracy of   car : 82.80 %\n",
      "Accuracy of  bird : 58.90 %\n",
      "Accuracy of   cat : 53.60 %\n",
      "Accuracy of  deer : 67.50 %\n",
      "Accuracy of   dog : 62.70 %\n",
      "Accuracy of  frog : 82.30 %\n",
      "Accuracy of horse : 78.70 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 81.30 %\n",
      "72.87\n",
      "[Epoch : 81, Iter:    50] loss: 0.004\n",
      "[Epoch : 82, Iter:    50] loss: 0.004\n",
      "[Epoch : 83, Iter:    50] loss: 0.004\n",
      "[Epoch : 84, Iter:    50] loss: 0.004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 85, Iter:    50] loss: 0.004\n",
      "[Epoch : 86, Iter:    50] loss: 0.004\n",
      "[Epoch : 87, Iter:    50] loss: 0.004\n",
      "[Epoch : 88, Iter:    50] loss: 0.004\n",
      "[Epoch : 89, Iter:    50] loss: 0.004\n",
      "[Epoch : 90, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 77.80 %\n",
      "Accuracy of   car : 81.40 %\n",
      "Accuracy of  bird : 61.00 %\n",
      "Accuracy of   cat : 52.80 %\n",
      "Accuracy of  deer : 70.20 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.60 %\n",
      "Accuracy of horse : 77.40 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 80.50 %\n",
      "72.57000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 82.10 %\n",
      "Accuracy of  bird : 59.20 %\n",
      "Accuracy of   cat : 54.10 %\n",
      "Accuracy of  deer : 68.50 %\n",
      "Accuracy of   dog : 62.00 %\n",
      "Accuracy of  frog : 81.90 %\n",
      "Accuracy of horse : 78.00 %\n",
      "Accuracy of  ship : 82.00 %\n",
      "Accuracy of truck : 81.10 %\n",
      "72.78\n",
      "[Epoch : 91, Iter:    50] loss: 0.004\n",
      "[Epoch : 92, Iter:    50] loss: 0.004\n",
      "[Epoch : 93, Iter:    50] loss: 0.004\n",
      "[Epoch : 94, Iter:    50] loss: 0.004\n",
      "[Epoch : 95, Iter:    50] loss: 0.004\n",
      "[Epoch : 96, Iter:    50] loss: 0.004\n",
      "[Epoch : 97, Iter:    50] loss: 0.004\n",
      "[Epoch : 98, Iter:    50] loss: 0.004\n",
      "[Epoch : 99, Iter:    50] loss: 0.004\n",
      "[Epoch : 100, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.50 %\n",
      "Accuracy of   car : 81.50 %\n",
      "Accuracy of  bird : 61.10 %\n",
      "Accuracy of   cat : 52.90 %\n",
      "Accuracy of  deer : 69.80 %\n",
      "Accuracy of   dog : 60.60 %\n",
      "Accuracy of  frog : 81.30 %\n",
      "Accuracy of horse : 77.30 %\n",
      "Accuracy of  ship : 82.40 %\n",
      "Accuracy of truck : 80.10 %\n",
      "72.65\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.70 %\n",
      "Accuracy of  bird : 59.80 %\n",
      "Accuracy of   cat : 53.90 %\n",
      "Accuracy of  deer : 68.90 %\n",
      "Accuracy of   dog : 61.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.80 %\n",
      "Accuracy of  ship : 81.90 %\n",
      "Accuracy of truck : 81.00 %\n",
      "72.74999999999999\n",
      "[Epoch : 101, Iter:    50] loss: 0.004\n",
      "[Epoch : 102, Iter:    50] loss: 0.004\n",
      "[Epoch : 103, Iter:    50] loss: 0.004\n",
      "[Epoch : 104, Iter:    50] loss: 0.004\n",
      "[Epoch : 105, Iter:    50] loss: 0.004\n",
      "[Epoch : 106, Iter:    50] loss: 0.004\n",
      "[Epoch : 107, Iter:    50] loss: 0.004\n",
      "[Epoch : 108, Iter:    50] loss: 0.004\n",
      "[Epoch : 109, Iter:    50] loss: 0.004\n",
      "[Epoch : 110, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 80.80 %\n",
      "Accuracy of  bird : 61.60 %\n",
      "Accuracy of   cat : 51.80 %\n",
      "Accuracy of  deer : 69.40 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 82.00 %\n",
      "Accuracy of horse : 77.10 %\n",
      "Accuracy of  ship : 81.50 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.46\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.40 %\n",
      "Accuracy of  bird : 60.00 %\n",
      "Accuracy of   cat : 53.50 %\n",
      "Accuracy of  deer : 69.00 %\n",
      "Accuracy of   dog : 61.20 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 81.80 %\n",
      "Accuracy of truck : 80.90 %\n",
      "72.6\n",
      "[Epoch : 111, Iter:    50] loss: 0.004\n",
      "[Epoch : 112, Iter:    50] loss: 0.004\n",
      "[Epoch : 113, Iter:    50] loss: 0.004\n",
      "[Epoch : 114, Iter:    50] loss: 0.004\n",
      "[Epoch : 115, Iter:    50] loss: 0.004\n",
      "[Epoch : 116, Iter:    50] loss: 0.004\n",
      "[Epoch : 117, Iter:    50] loss: 0.004\n",
      "[Epoch : 118, Iter:    50] loss: 0.004\n",
      "[Epoch : 119, Iter:    50] loss: 0.004\n",
      "[Epoch : 120, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.00 %\n",
      "Accuracy of   car : 81.50 %\n",
      "Accuracy of  bird : 60.60 %\n",
      "Accuracy of   cat : 52.50 %\n",
      "Accuracy of  deer : 69.60 %\n",
      "Accuracy of   dog : 61.10 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.40 %\n",
      "Accuracy of  ship : 82.00 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.58\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.80 %\n",
      "Accuracy of   car : 81.40 %\n",
      "Accuracy of  bird : 60.10 %\n",
      "Accuracy of   cat : 53.20 %\n",
      "Accuracy of  deer : 69.20 %\n",
      "Accuracy of   dog : 61.30 %\n",
      "Accuracy of  frog : 81.60 %\n",
      "Accuracy of horse : 77.30 %\n",
      "Accuracy of  ship : 81.90 %\n",
      "Accuracy of truck : 80.80 %\n",
      "72.55999999999999\n",
      "[Epoch : 121, Iter:    50] loss: 0.004\n",
      "[Epoch : 122, Iter:    50] loss: 0.004\n",
      "[Epoch : 123, Iter:    50] loss: 0.004\n",
      "[Epoch : 124, Iter:    50] loss: 0.004\n",
      "[Epoch : 125, Iter:    50] loss: 0.004\n",
      "[Epoch : 126, Iter:    50] loss: 0.004\n",
      "[Epoch : 127, Iter:    50] loss: 0.004\n",
      "[Epoch : 128, Iter:    50] loss: 0.004\n",
      "[Epoch : 129, Iter:    50] loss: 0.004\n",
      "[Epoch : 130, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.40 %\n",
      "Accuracy of   car : 81.10 %\n",
      "Accuracy of  bird : 61.40 %\n",
      "Accuracy of   cat : 52.20 %\n",
      "Accuracy of  deer : 69.60 %\n",
      "Accuracy of   dog : 60.60 %\n",
      "Accuracy of  frog : 81.70 %\n",
      "Accuracy of horse : 77.20 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.50 %\n",
      "72.5\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.40 %\n",
      "Accuracy of  bird : 60.00 %\n",
      "Accuracy of   cat : 53.00 %\n",
      "Accuracy of  deer : 69.40 %\n",
      "Accuracy of   dog : 61.10 %\n",
      "Accuracy of  frog : 81.70 %\n",
      "Accuracy of horse : 77.20 %\n",
      "Accuracy of  ship : 82.10 %\n",
      "Accuracy of truck : 80.50 %\n",
      "72.53\n",
      "[Epoch : 131, Iter:    50] loss: 0.004\n",
      "[Epoch : 132, Iter:    50] loss: 0.004\n",
      "[Epoch : 133, Iter:    50] loss: 0.004\n",
      "[Epoch : 134, Iter:    50] loss: 0.004\n",
      "[Epoch : 135, Iter:    50] loss: 0.004\n",
      "[Epoch : 136, Iter:    50] loss: 0.004\n",
      "[Epoch : 137, Iter:    50] loss: 0.004\n",
      "[Epoch : 138, Iter:    50] loss: 0.004\n",
      "[Epoch : 139, Iter:    50] loss: 0.004\n",
      "[Epoch : 140, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 81.00 %\n",
      "Accuracy of  bird : 60.60 %\n",
      "Accuracy of   cat : 52.30 %\n",
      "Accuracy of  deer : 69.80 %\n",
      "Accuracy of   dog : 60.70 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.40 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.52000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.00 %\n",
      "Accuracy of  bird : 60.10 %\n",
      "Accuracy of   cat : 52.90 %\n",
      "Accuracy of  deer : 69.50 %\n",
      "Accuracy of   dog : 61.10 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.20 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.5\n",
      "[Epoch : 141, Iter:    50] loss: 0.004\n",
      "[Epoch : 142, Iter:    50] loss: 0.004\n",
      "[Epoch : 143, Iter:    50] loss: 0.004\n",
      "[Epoch : 144, Iter:    50] loss: 0.004\n",
      "[Epoch : 145, Iter:    50] loss: 0.004\n",
      "[Epoch : 146, Iter:    50] loss: 0.004\n",
      "[Epoch : 147, Iter:    50] loss: 0.004\n",
      "[Epoch : 148, Iter:    50] loss: 0.004\n",
      "[Epoch : 149, Iter:    50] loss: 0.004\n",
      "[Epoch : 150, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.40 %\n",
      "Accuracy of   cat : 52.20 %\n",
      "Accuracy of  deer : 69.90 %\n",
      "Accuracy of   dog : 60.90 %\n",
      "Accuracy of  frog : 81.60 %\n",
      "Accuracy of horse : 77.40 %\n",
      "Accuracy of  ship : 82.10 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.53\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 80.90 %\n",
      "Accuracy of  bird : 60.50 %\n",
      "Accuracy of   cat : 52.60 %\n",
      "Accuracy of  deer : 69.60 %\n",
      "Accuracy of   dog : 60.90 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.10 %\n",
      "Accuracy of  ship : 82.10 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.5\n",
      "[Epoch : 151, Iter:    50] loss: 0.004\n",
      "[Epoch : 152, Iter:    50] loss: 0.004\n",
      "[Epoch : 153, Iter:    50] loss: 0.004\n",
      "[Epoch : 154, Iter:    50] loss: 0.004\n",
      "[Epoch : 155, Iter:    50] loss: 0.004\n",
      "[Epoch : 156, Iter:    50] loss: 0.004\n",
      "[Epoch : 157, Iter:    50] loss: 0.004\n",
      "[Epoch : 158, Iter:    50] loss: 0.004\n",
      "[Epoch : 159, Iter:    50] loss: 0.004\n",
      "[Epoch : 160, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.80 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 61.10 %\n",
      "Accuracy of   cat : 52.10 %\n",
      "Accuracy of  deer : 69.90 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.70 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.50 %\n",
      "72.58999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 81.00 %\n",
      "Accuracy of  bird : 60.50 %\n",
      "Accuracy of   cat : 52.50 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.30 %\n",
      "Accuracy of  ship : 82.20 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.53\n",
      "[Epoch : 161, Iter:    50] loss: 0.004\n",
      "[Epoch : 162, Iter:    50] loss: 0.004\n",
      "[Epoch : 163, Iter:    50] loss: 0.004\n",
      "[Epoch : 164, Iter:    50] loss: 0.004\n",
      "[Epoch : 165, Iter:    50] loss: 0.004\n",
      "[Epoch : 166, Iter:    50] loss: 0.004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 167, Iter:    50] loss: 0.004\n",
      "[Epoch : 168, Iter:    50] loss: 0.004\n",
      "[Epoch : 169, Iter:    50] loss: 0.004\n",
      "[Epoch : 170, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.80 %\n",
      "Accuracy of   cat : 52.10 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.53999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 81.00 %\n",
      "Accuracy of  bird : 60.60 %\n",
      "Accuracy of   cat : 52.50 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.90 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.30 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.55999999999999\n",
      "[Epoch : 171, Iter:    50] loss: 0.004\n",
      "[Epoch : 172, Iter:    50] loss: 0.004\n",
      "[Epoch : 173, Iter:    50] loss: 0.004\n",
      "[Epoch : 174, Iter:    50] loss: 0.004\n",
      "[Epoch : 175, Iter:    50] loss: 0.004\n",
      "[Epoch : 176, Iter:    50] loss: 0.004\n",
      "[Epoch : 177, Iter:    50] loss: 0.004\n",
      "[Epoch : 178, Iter:    50] loss: 0.004\n",
      "[Epoch : 179, Iter:    50] loss: 0.004\n",
      "[Epoch : 180, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.90 %\n",
      "Accuracy of   car : 81.10 %\n",
      "Accuracy of  bird : 60.90 %\n",
      "Accuracy of   cat : 52.40 %\n",
      "Accuracy of  deer : 69.60 %\n",
      "Accuracy of   dog : 60.70 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.40 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.57000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.70 %\n",
      "Accuracy of   cat : 52.40 %\n",
      "Accuracy of  deer : 69.80 %\n",
      "Accuracy of   dog : 60.90 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.60999999999999\n",
      "[Epoch : 181, Iter:    50] loss: 0.004\n",
      "[Epoch : 182, Iter:    50] loss: 0.004\n",
      "[Epoch : 183, Iter:    50] loss: 0.004\n",
      "[Epoch : 184, Iter:    50] loss: 0.004\n",
      "[Epoch : 185, Iter:    50] loss: 0.004\n",
      "[Epoch : 186, Iter:    50] loss: 0.004\n",
      "[Epoch : 187, Iter:    50] loss: 0.004\n",
      "[Epoch : 188, Iter:    50] loss: 0.004\n",
      "[Epoch : 189, Iter:    50] loss: 0.004\n",
      "[Epoch : 190, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.00 %\n",
      "Accuracy of   car : 81.30 %\n",
      "Accuracy of  bird : 60.80 %\n",
      "Accuracy of   cat : 52.30 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.40 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.30 %\n",
      "72.56999999999998\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.70 %\n",
      "Accuracy of   cat : 52.20 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.57\n",
      "[Epoch : 191, Iter:    50] loss: 0.004\n",
      "[Epoch : 192, Iter:    50] loss: 0.004\n",
      "[Epoch : 193, Iter:    50] loss: 0.004\n",
      "[Epoch : 194, Iter:    50] loss: 0.004\n",
      "[Epoch : 195, Iter:    50] loss: 0.004\n",
      "[Epoch : 196, Iter:    50] loss: 0.004\n",
      "[Epoch : 197, Iter:    50] loss: 0.004\n",
      "[Epoch : 198, Iter:    50] loss: 0.004\n",
      "[Epoch : 199, Iter:    50] loss: 0.004\n",
      "[Epoch : 200, Iter:    50] loss: 0.004\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.80 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.90 %\n",
      "Accuracy of   cat : 52.20 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.60 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.50 %\n",
      "Accuracy of truck : 80.50 %\n",
      "72.57000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.00 %\n",
      "Accuracy of   car : 81.20 %\n",
      "Accuracy of  bird : 60.80 %\n",
      "Accuracy of   cat : 52.20 %\n",
      "Accuracy of  deer : 69.70 %\n",
      "Accuracy of   dog : 60.80 %\n",
      "Accuracy of  frog : 81.80 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 82.30 %\n",
      "Accuracy of truck : 80.40 %\n",
      "72.57\n",
      "\n",
      "Net_DO\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a904404a75584bb1a55ca0961a604760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=200), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 1.656\n",
      "[Epoch : 2, Iter:    50] loss: 1.195\n",
      "[Epoch : 3, Iter:    50] loss: 0.978\n",
      "[Epoch : 4, Iter:    50] loss: 0.845\n",
      "[Epoch : 5, Iter:    50] loss: 0.728\n",
      "[Epoch : 6, Iter:    50] loss: 0.614\n",
      "[Epoch : 7, Iter:    50] loss: 0.521\n",
      "[Epoch : 8, Iter:    50] loss: 0.432\n",
      "[Epoch : 9, Iter:    50] loss: 0.359\n",
      "[Epoch : 10, Iter:    50] loss: 0.294\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 86.10 %\n",
      "Accuracy of  bird : 56.60 %\n",
      "Accuracy of   cat : 53.60 %\n",
      "Accuracy of  deer : 62.70 %\n",
      "Accuracy of   dog : 67.40 %\n",
      "Accuracy of  frog : 83.40 %\n",
      "Accuracy of horse : 78.70 %\n",
      "Accuracy of  ship : 81.30 %\n",
      "Accuracy of truck : 80.60 %\n",
      "72.96000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 66.00 %\n",
      "Accuracy of   car : 44.30 %\n",
      "Accuracy of  bird : 24.50 %\n",
      "Accuracy of   cat : 41.20 %\n",
      "Accuracy of  deer : 33.70 %\n",
      "Accuracy of   dog : 53.10 %\n",
      "Accuracy of  frog : 53.80 %\n",
      "Accuracy of horse : 45.60 %\n",
      "Accuracy of  ship : 68.60 %\n",
      "Accuracy of truck : 80.40 %\n",
      "51.120000000000005\n",
      "[Epoch : 11, Iter:    50] loss: 0.213\n",
      "[Epoch : 12, Iter:    50] loss: 0.163\n",
      "[Epoch : 13, Iter:    50] loss: 0.138\n",
      "[Epoch : 14, Iter:    50] loss: 0.127\n",
      "[Epoch : 15, Iter:    50] loss: 0.118\n",
      "[Epoch : 16, Iter:    50] loss: 0.107\n",
      "[Epoch : 17, Iter:    50] loss: 0.104\n",
      "[Epoch : 18, Iter:    50] loss: 0.098\n",
      "[Epoch : 19, Iter:    50] loss: 0.095\n",
      "[Epoch : 20, Iter:    50] loss: 0.091\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 78.60 %\n",
      "Accuracy of   car : 83.00 %\n",
      "Accuracy of  bird : 68.70 %\n",
      "Accuracy of   cat : 56.40 %\n",
      "Accuracy of  deer : 63.20 %\n",
      "Accuracy of   dog : 65.50 %\n",
      "Accuracy of  frog : 76.10 %\n",
      "Accuracy of horse : 79.70 %\n",
      "Accuracy of  ship : 84.20 %\n",
      "Accuracy of truck : 82.60 %\n",
      "73.80000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 76.80 %\n",
      "Accuracy of   car : 75.70 %\n",
      "Accuracy of  bird : 41.30 %\n",
      "Accuracy of   cat : 48.20 %\n",
      "Accuracy of  deer : 53.60 %\n",
      "Accuracy of   dog : 62.00 %\n",
      "Accuracy of  frog : 79.50 %\n",
      "Accuracy of horse : 73.60 %\n",
      "Accuracy of  ship : 76.30 %\n",
      "Accuracy of truck : 88.80 %\n",
      "67.58\n",
      "[Epoch : 21, Iter:    50] loss: 0.071\n",
      "[Epoch : 22, Iter:    50] loss: 0.055\n",
      "[Epoch : 23, Iter:    50] loss: 0.046\n",
      "[Epoch : 24, Iter:    50] loss: 0.042\n",
      "[Epoch : 25, Iter:    50] loss: 0.040\n",
      "[Epoch : 26, Iter:    50] loss: 0.041\n",
      "[Epoch : 27, Iter:    50] loss: 0.040\n",
      "[Epoch : 28, Iter:    50] loss: 0.041\n",
      "[Epoch : 29, Iter:    50] loss: 0.044\n",
      "[Epoch : 30, Iter:    50] loss: 0.047\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.70 %\n",
      "Accuracy of   car : 81.60 %\n",
      "Accuracy of  bird : 64.40 %\n",
      "Accuracy of   cat : 50.80 %\n",
      "Accuracy of  deer : 73.90 %\n",
      "Accuracy of   dog : 64.60 %\n",
      "Accuracy of  frog : 80.10 %\n",
      "Accuracy of horse : 79.10 %\n",
      "Accuracy of  ship : 82.50 %\n",
      "Accuracy of truck : 84.10 %\n",
      "74.08000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 79.50 %\n",
      "Accuracy of   car : 83.50 %\n",
      "Accuracy of  bird : 53.10 %\n",
      "Accuracy of   cat : 52.50 %\n",
      "Accuracy of  deer : 65.30 %\n",
      "Accuracy of   dog : 65.60 %\n",
      "Accuracy of  frog : 84.20 %\n",
      "Accuracy of horse : 79.80 %\n",
      "Accuracy of  ship : 81.20 %\n",
      "Accuracy of truck : 87.60 %\n",
      "73.23\n",
      "[Epoch : 31, Iter:    50] loss: 0.037\n",
      "[Epoch : 32, Iter:    50] loss: 0.031\n",
      "[Epoch : 33, Iter:    50] loss: 0.030\n",
      "[Epoch : 34, Iter:    50] loss: 0.028\n",
      "[Epoch : 35, Iter:    50] loss: 0.024\n",
      "[Epoch : 36, Iter:    50] loss: 0.024\n",
      "[Epoch : 37, Iter:    50] loss: 0.024\n",
      "[Epoch : 38, Iter:    50] loss: 0.025\n",
      "[Epoch : 39, Iter:    50] loss: 0.024\n",
      "[Epoch : 40, Iter:    50] loss: 0.023\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 84.60 %\n",
      "Accuracy of  bird : 61.50 %\n",
      "Accuracy of   cat : 55.60 %\n",
      "Accuracy of  deer : 73.40 %\n",
      "Accuracy of   dog : 63.40 %\n",
      "Accuracy of  frog : 83.10 %\n",
      "Accuracy of horse : 82.80 %\n",
      "Accuracy of  ship : 82.80 %\n",
      "Accuracy of truck : 82.00 %\n",
      "74.83999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 80.40 %\n",
      "Accuracy of   car : 85.30 %\n",
      "Accuracy of  bird : 58.80 %\n",
      "Accuracy of   cat : 54.30 %\n",
      "Accuracy of  deer : 70.40 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 85.90 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 83.20 %\n",
      "Accuracy of truck : 86.20 %\n",
      "75.17\n",
      "[Epoch : 41, Iter:    50] loss: 0.024\n",
      "[Epoch : 42, Iter:    50] loss: 0.020\n",
      "[Epoch : 43, Iter:    50] loss: 0.018\n",
      "[Epoch : 44, Iter:    50] loss: 0.016\n",
      "[Epoch : 45, Iter:    50] loss: 0.017\n",
      "[Epoch : 46, Iter:    50] loss: 0.015\n",
      "[Epoch : 47, Iter:    50] loss: 0.016\n",
      "[Epoch : 48, Iter:    50] loss: 0.016\n",
      "[Epoch : 49, Iter:    50] loss: 0.015\n",
      "[Epoch : 50, Iter:    50] loss: 0.016\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 80.90 %\n",
      "Accuracy of   car : 84.20 %\n",
      "Accuracy of  bird : 60.20 %\n",
      "Accuracy of   cat : 59.00 %\n",
      "Accuracy of  deer : 69.80 %\n",
      "Accuracy of   dog : 62.90 %\n",
      "Accuracy of  frog : 81.40 %\n",
      "Accuracy of horse : 79.60 %\n",
      "Accuracy of  ship : 83.80 %\n",
      "Accuracy of truck : 84.10 %\n",
      "74.59\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 80.40 %\n",
      "Accuracy of   car : 85.50 %\n",
      "Accuracy of  bird : 61.10 %\n",
      "Accuracy of   cat : 55.30 %\n",
      "Accuracy of  deer : 72.80 %\n",
      "Accuracy of   dog : 66.90 %\n",
      "Accuracy of  frog : 85.70 %\n",
      "Accuracy of horse : 80.80 %\n",
      "Accuracy of  ship : 84.40 %\n",
      "Accuracy of truck : 85.50 %\n",
      "75.84\n",
      "[Epoch : 51, Iter:    50] loss: 0.015\n",
      "[Epoch : 52, Iter:    50] loss: 0.013\n",
      "[Epoch : 53, Iter:    50] loss: 0.012\n",
      "[Epoch : 54, Iter:    50] loss: 0.011\n",
      "[Epoch : 55, Iter:    50] loss: 0.011\n",
      "[Epoch : 56, Iter:    50] loss: 0.012\n",
      "[Epoch : 57, Iter:    50] loss: 0.011\n",
      "[Epoch : 58, Iter:    50] loss: 0.011\n",
      "[Epoch : 59, Iter:    50] loss: 0.011\n",
      "[Epoch : 60, Iter:    50] loss: 0.011\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 79.80 %\n",
      "Accuracy of   car : 85.40 %\n",
      "Accuracy of  bird : 61.90 %\n",
      "Accuracy of   cat : 56.00 %\n",
      "Accuracy of  deer : 71.50 %\n",
      "Accuracy of   dog : 64.30 %\n",
      "Accuracy of  frog : 84.00 %\n",
      "Accuracy of horse : 80.90 %\n",
      "Accuracy of  ship : 85.40 %\n",
      "Accuracy of truck : 81.70 %\n",
      "75.09\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.00 %\n",
      "Accuracy of   car : 85.80 %\n",
      "Accuracy of  bird : 63.20 %\n",
      "Accuracy of   cat : 54.80 %\n",
      "Accuracy of  deer : 73.00 %\n",
      "Accuracy of   dog : 67.50 %\n",
      "Accuracy of  frog : 85.40 %\n",
      "Accuracy of horse : 81.00 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 84.50 %\n",
      "76.11\n",
      "[Epoch : 61, Iter:    50] loss: 0.010\n",
      "[Epoch : 62, Iter:    50] loss: 0.010\n",
      "[Epoch : 63, Iter:    50] loss: 0.010\n",
      "[Epoch : 64, Iter:    50] loss: 0.010\n",
      "[Epoch : 65, Iter:    50] loss: 0.009\n",
      "[Epoch : 66, Iter:    50] loss: 0.009\n",
      "[Epoch : 67, Iter:    50] loss: 0.009\n",
      "[Epoch : 68, Iter:    50] loss: 0.009\n",
      "[Epoch : 69, Iter:    50] loss: 0.009\n",
      "[Epoch : 70, Iter:    50] loss: 0.008\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.50 %\n",
      "Accuracy of   car : 85.60 %\n",
      "Accuracy of  bird : 66.40 %\n",
      "Accuracy of   cat : 56.60 %\n",
      "Accuracy of  deer : 71.90 %\n",
      "Accuracy of   dog : 62.30 %\n",
      "Accuracy of  frog : 83.40 %\n",
      "Accuracy of horse : 80.60 %\n",
      "Accuracy of  ship : 84.30 %\n",
      "Accuracy of truck : 82.20 %\n",
      "75.47999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 80.90 %\n",
      "Accuracy of   car : 86.40 %\n",
      "Accuracy of  bird : 64.10 %\n",
      "Accuracy of   cat : 55.00 %\n",
      "Accuracy of  deer : 73.10 %\n",
      "Accuracy of   dog : 67.10 %\n",
      "Accuracy of  frog : 84.50 %\n",
      "Accuracy of horse : 81.40 %\n",
      "Accuracy of  ship : 85.10 %\n",
      "Accuracy of truck : 84.00 %\n",
      "76.16\n",
      "[Epoch : 71, Iter:    50] loss: 0.009\n",
      "[Epoch : 72, Iter:    50] loss: 0.009\n",
      "[Epoch : 73, Iter:    50] loss: 0.008\n",
      "[Epoch : 74, Iter:    50] loss: 0.008\n",
      "[Epoch : 75, Iter:    50] loss: 0.008\n",
      "[Epoch : 76, Iter:    50] loss: 0.008\n",
      "[Epoch : 77, Iter:    50] loss: 0.008\n",
      "[Epoch : 78, Iter:    50] loss: 0.008\n",
      "[Epoch : 79, Iter:    50] loss: 0.008\n",
      "[Epoch : 80, Iter:    50] loss: 0.008\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.50 %\n",
      "Accuracy of   car : 84.60 %\n",
      "Accuracy of  bird : 65.70 %\n",
      "Accuracy of   cat : 55.30 %\n",
      "Accuracy of  deer : 73.30 %\n",
      "Accuracy of   dog : 65.50 %\n",
      "Accuracy of  frog : 83.40 %\n",
      "Accuracy of horse : 79.50 %\n",
      "Accuracy of  ship : 85.50 %\n",
      "Accuracy of truck : 82.60 %\n",
      "75.69000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.30 %\n",
      "Accuracy of   car : 86.20 %\n",
      "Accuracy of  bird : 64.30 %\n",
      "Accuracy of   cat : 55.30 %\n",
      "Accuracy of  deer : 72.50 %\n",
      "Accuracy of   dog : 66.80 %\n",
      "Accuracy of  frog : 84.30 %\n",
      "Accuracy of horse : 81.10 %\n",
      "Accuracy of  ship : 85.00 %\n",
      "Accuracy of truck : 83.20 %\n",
      "76.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 81, Iter:    50] loss: 0.008\n",
      "[Epoch : 82, Iter:    50] loss: 0.008\n",
      "[Epoch : 83, Iter:    50] loss: 0.008\n",
      "[Epoch : 84, Iter:    50] loss: 0.008\n",
      "[Epoch : 85, Iter:    50] loss: 0.008\n",
      "[Epoch : 86, Iter:    50] loss: 0.008\n",
      "[Epoch : 87, Iter:    50] loss: 0.007\n",
      "[Epoch : 88, Iter:    50] loss: 0.008\n",
      "[Epoch : 89, Iter:    50] loss: 0.008\n",
      "[Epoch : 90, Iter:    50] loss: 0.008\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.30 %\n",
      "Accuracy of   car : 86.20 %\n",
      "Accuracy of  bird : 65.40 %\n",
      "Accuracy of   cat : 57.70 %\n",
      "Accuracy of  deer : 71.30 %\n",
      "Accuracy of   dog : 66.30 %\n",
      "Accuracy of  frog : 83.00 %\n",
      "Accuracy of horse : 80.40 %\n",
      "Accuracy of  ship : 85.30 %\n",
      "Accuracy of truck : 82.10 %\n",
      "75.9\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.00 %\n",
      "Accuracy of   car : 86.50 %\n",
      "Accuracy of  bird : 64.30 %\n",
      "Accuracy of   cat : 56.10 %\n",
      "Accuracy of  deer : 72.90 %\n",
      "Accuracy of   dog : 66.80 %\n",
      "Accuracy of  frog : 84.30 %\n",
      "Accuracy of horse : 81.00 %\n",
      "Accuracy of  ship : 85.10 %\n",
      "Accuracy of truck : 83.00 %\n",
      "76.1\n",
      "[Epoch : 91, Iter:    50] loss: 0.007\n",
      "[Epoch : 92, Iter:    50] loss: 0.007\n",
      "[Epoch : 93, Iter:    50] loss: 0.007\n",
      "[Epoch : 94, Iter:    50] loss: 0.007\n",
      "[Epoch : 95, Iter:    50] loss: 0.007\n",
      "[Epoch : 96, Iter:    50] loss: 0.007\n",
      "[Epoch : 97, Iter:    50] loss: 0.008\n",
      "[Epoch : 98, Iter:    50] loss: 0.007\n",
      "[Epoch : 99, Iter:    50] loss: 0.007\n",
      "[Epoch : 100, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 85.70 %\n",
      "Accuracy of  bird : 64.70 %\n",
      "Accuracy of   cat : 56.70 %\n",
      "Accuracy of  deer : 72.60 %\n",
      "Accuracy of   dog : 66.40 %\n",
      "Accuracy of  frog : 82.10 %\n",
      "Accuracy of horse : 80.00 %\n",
      "Accuracy of  ship : 84.70 %\n",
      "Accuracy of truck : 83.50 %\n",
      "75.82000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.30 %\n",
      "Accuracy of   car : 86.30 %\n",
      "Accuracy of  bird : 64.30 %\n",
      "Accuracy of   cat : 56.20 %\n",
      "Accuracy of  deer : 72.60 %\n",
      "Accuracy of   dog : 66.70 %\n",
      "Accuracy of  frog : 84.20 %\n",
      "Accuracy of horse : 80.80 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 83.00 %\n",
      "76.03\n",
      "[Epoch : 101, Iter:    50] loss: 0.007\n",
      "[Epoch : 102, Iter:    50] loss: 0.007\n",
      "[Epoch : 103, Iter:    50] loss: 0.007\n",
      "[Epoch : 104, Iter:    50] loss: 0.007\n",
      "[Epoch : 105, Iter:    50] loss: 0.007\n",
      "[Epoch : 106, Iter:    50] loss: 0.007\n",
      "[Epoch : 107, Iter:    50] loss: 0.007\n",
      "[Epoch : 108, Iter:    50] loss: 0.007\n",
      "[Epoch : 109, Iter:    50] loss: 0.007\n",
      "[Epoch : 110, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.50 %\n",
      "Accuracy of   car : 86.10 %\n",
      "Accuracy of  bird : 65.20 %\n",
      "Accuracy of   cat : 56.10 %\n",
      "Accuracy of  deer : 72.80 %\n",
      "Accuracy of   dog : 66.10 %\n",
      "Accuracy of  frog : 84.10 %\n",
      "Accuracy of horse : 81.10 %\n",
      "Accuracy of  ship : 84.60 %\n",
      "Accuracy of truck : 82.30 %\n",
      "75.99\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.30 %\n",
      "Accuracy of   car : 86.20 %\n",
      "Accuracy of  bird : 64.20 %\n",
      "Accuracy of   cat : 56.30 %\n",
      "Accuracy of  deer : 72.80 %\n",
      "Accuracy of   dog : 66.40 %\n",
      "Accuracy of  frog : 84.00 %\n",
      "Accuracy of horse : 80.80 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 83.10 %\n",
      "76.0\n",
      "[Epoch : 111, Iter:    50] loss: 0.007\n",
      "[Epoch : 112, Iter:    50] loss: 0.007\n",
      "[Epoch : 113, Iter:    50] loss: 0.007\n",
      "[Epoch : 114, Iter:    50] loss: 0.007\n",
      "[Epoch : 115, Iter:    50] loss: 0.007\n",
      "[Epoch : 116, Iter:    50] loss: 0.007\n",
      "[Epoch : 117, Iter:    50] loss: 0.007\n",
      "[Epoch : 118, Iter:    50] loss: 0.007\n",
      "[Epoch : 119, Iter:    50] loss: 0.007\n",
      "[Epoch : 120, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.70 %\n",
      "Accuracy of   car : 85.40 %\n",
      "Accuracy of  bird : 64.90 %\n",
      "Accuracy of   cat : 56.60 %\n",
      "Accuracy of  deer : 72.40 %\n",
      "Accuracy of   dog : 66.70 %\n",
      "Accuracy of  frog : 83.30 %\n",
      "Accuracy of horse : 80.10 %\n",
      "Accuracy of  ship : 84.80 %\n",
      "Accuracy of truck : 83.10 %\n",
      "75.9\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.50 %\n",
      "Accuracy of   car : 86.50 %\n",
      "Accuracy of  bird : 64.50 %\n",
      "Accuracy of   cat : 56.60 %\n",
      "Accuracy of  deer : 72.60 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 83.90 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 85.10 %\n",
      "Accuracy of truck : 82.90 %\n",
      "76.08000000000001\n",
      "[Epoch : 121, Iter:    50] loss: 0.007\n",
      "[Epoch : 122, Iter:    50] loss: 0.007\n",
      "[Epoch : 123, Iter:    50] loss: 0.007\n",
      "[Epoch : 124, Iter:    50] loss: 0.007\n",
      "[Epoch : 125, Iter:    50] loss: 0.007\n",
      "[Epoch : 126, Iter:    50] loss: 0.007\n",
      "[Epoch : 127, Iter:    50] loss: 0.007\n",
      "[Epoch : 128, Iter:    50] loss: 0.007\n",
      "[Epoch : 129, Iter:    50] loss: 0.007\n",
      "[Epoch : 130, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 86.40 %\n",
      "Accuracy of  bird : 64.70 %\n",
      "Accuracy of   cat : 55.60 %\n",
      "Accuracy of  deer : 73.40 %\n",
      "Accuracy of   dog : 65.70 %\n",
      "Accuracy of  frog : 84.50 %\n",
      "Accuracy of horse : 81.00 %\n",
      "Accuracy of  ship : 84.60 %\n",
      "Accuracy of truck : 82.30 %\n",
      "76.0\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.90 %\n",
      "Accuracy of   car : 86.30 %\n",
      "Accuracy of  bird : 64.90 %\n",
      "Accuracy of   cat : 56.20 %\n",
      "Accuracy of  deer : 72.60 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 84.10 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 83.00 %\n",
      "76.11\n",
      "[Epoch : 131, Iter:    50] loss: 0.007\n",
      "[Epoch : 132, Iter:    50] loss: 0.007\n",
      "[Epoch : 133, Iter:    50] loss: 0.007\n",
      "[Epoch : 134, Iter:    50] loss: 0.007\n",
      "[Epoch : 135, Iter:    50] loss: 0.007\n",
      "[Epoch : 136, Iter:    50] loss: 0.007\n",
      "[Epoch : 137, Iter:    50] loss: 0.007\n",
      "[Epoch : 138, Iter:    50] loss: 0.007\n",
      "[Epoch : 139, Iter:    50] loss: 0.007\n",
      "[Epoch : 140, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.60 %\n",
      "Accuracy of   car : 85.90 %\n",
      "Accuracy of  bird : 65.50 %\n",
      "Accuracy of   cat : 55.60 %\n",
      "Accuracy of  deer : 73.40 %\n",
      "Accuracy of   dog : 65.70 %\n",
      "Accuracy of  frog : 84.10 %\n",
      "Accuracy of horse : 80.00 %\n",
      "Accuracy of  ship : 85.40 %\n",
      "Accuracy of truck : 82.80 %\n",
      "76.0\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.70 %\n",
      "Accuracy of   car : 86.30 %\n",
      "Accuracy of  bird : 65.00 %\n",
      "Accuracy of   cat : 56.30 %\n",
      "Accuracy of  deer : 72.90 %\n",
      "Accuracy of   dog : 66.60 %\n",
      "Accuracy of  frog : 83.90 %\n",
      "Accuracy of horse : 80.90 %\n",
      "Accuracy of  ship : 85.20 %\n",
      "Accuracy of truck : 83.00 %\n",
      "76.18\n",
      "[Epoch : 141, Iter:    50] loss: 0.007\n",
      "[Epoch : 142, Iter:    50] loss: 0.007\n",
      "[Epoch : 143, Iter:    50] loss: 0.007\n",
      "[Epoch : 144, Iter:    50] loss: 0.007\n",
      "[Epoch : 145, Iter:    50] loss: 0.007\n",
      "[Epoch : 146, Iter:    50] loss: 0.007\n",
      "[Epoch : 147, Iter:    50] loss: 0.007\n",
      "[Epoch : 148, Iter:    50] loss: 0.007\n",
      "[Epoch : 149, Iter:    50] loss: 0.007\n",
      "[Epoch : 150, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 86.00 %\n",
      "Accuracy of  bird : 65.00 %\n",
      "Accuracy of   cat : 55.90 %\n",
      "Accuracy of  deer : 73.20 %\n",
      "Accuracy of   dog : 66.00 %\n",
      "Accuracy of  frog : 84.30 %\n",
      "Accuracy of horse : 80.60 %\n",
      "Accuracy of  ship : 84.80 %\n",
      "Accuracy of truck : 83.50 %\n",
      "76.10999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 86.10 %\n",
      "Accuracy of  bird : 65.40 %\n",
      "Accuracy of   cat : 56.00 %\n",
      "Accuracy of  deer : 73.20 %\n",
      "Accuracy of   dog : 66.40 %\n",
      "Accuracy of  frog : 83.60 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 85.50 %\n",
      "Accuracy of truck : 83.20 %\n",
      "76.19000000000001\n",
      "[Epoch : 151, Iter:    50] loss: 0.007\n",
      "[Epoch : 152, Iter:    50] loss: 0.007\n",
      "[Epoch : 153, Iter:    50] loss: 0.007\n",
      "[Epoch : 154, Iter:    50] loss: 0.007\n",
      "[Epoch : 155, Iter:    50] loss: 0.007\n",
      "[Epoch : 156, Iter:    50] loss: 0.007\n",
      "[Epoch : 157, Iter:    50] loss: 0.007\n",
      "[Epoch : 158, Iter:    50] loss: 0.006\n",
      "[Epoch : 159, Iter:    50] loss: 0.007\n",
      "[Epoch : 160, Iter:    50] loss: 0.006\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 85.60 %\n",
      "Accuracy of  bird : 64.90 %\n",
      "Accuracy of   cat : 55.80 %\n",
      "Accuracy of  deer : 71.70 %\n",
      "Accuracy of   dog : 66.70 %\n",
      "Accuracy of  frog : 83.60 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 83.20 %\n",
      "75.89\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.90 %\n",
      "Accuracy of   car : 85.80 %\n",
      "Accuracy of  bird : 65.30 %\n",
      "Accuracy of   cat : 56.00 %\n",
      "Accuracy of  deer : 72.80 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 83.50 %\n",
      "Accuracy of horse : 80.60 %\n",
      "Accuracy of  ship : 85.50 %\n",
      "Accuracy of truck : 83.10 %\n",
      "76.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 161, Iter:    50] loss: 0.007\n",
      "[Epoch : 162, Iter:    50] loss: 0.007\n",
      "[Epoch : 163, Iter:    50] loss: 0.007\n",
      "[Epoch : 164, Iter:    50] loss: 0.007\n",
      "[Epoch : 165, Iter:    50] loss: 0.007\n",
      "[Epoch : 166, Iter:    50] loss: 0.006\n",
      "[Epoch : 167, Iter:    50] loss: 0.007\n",
      "[Epoch : 168, Iter:    50] loss: 0.007\n",
      "[Epoch : 169, Iter:    50] loss: 0.007\n",
      "[Epoch : 170, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 82.00 %\n",
      "Accuracy of   car : 85.00 %\n",
      "Accuracy of  bird : 64.90 %\n",
      "Accuracy of   cat : 55.90 %\n",
      "Accuracy of  deer : 73.30 %\n",
      "Accuracy of   dog : 66.20 %\n",
      "Accuracy of  frog : 83.70 %\n",
      "Accuracy of horse : 80.40 %\n",
      "Accuracy of  ship : 85.20 %\n",
      "Accuracy of truck : 83.00 %\n",
      "75.96000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.90 %\n",
      "Accuracy of   car : 85.70 %\n",
      "Accuracy of  bird : 65.20 %\n",
      "Accuracy of   cat : 56.20 %\n",
      "Accuracy of  deer : 73.00 %\n",
      "Accuracy of   dog : 66.40 %\n",
      "Accuracy of  frog : 83.60 %\n",
      "Accuracy of horse : 80.80 %\n",
      "Accuracy of  ship : 85.30 %\n",
      "Accuracy of truck : 82.90 %\n",
      "76.1\n",
      "[Epoch : 171, Iter:    50] loss: 0.007\n",
      "[Epoch : 172, Iter:    50] loss: 0.007\n",
      "[Epoch : 173, Iter:    50] loss: 0.007\n",
      "[Epoch : 174, Iter:    50] loss: 0.007\n",
      "[Epoch : 175, Iter:    50] loss: 0.007\n",
      "[Epoch : 176, Iter:    50] loss: 0.007\n",
      "[Epoch : 177, Iter:    50] loss: 0.007\n",
      "[Epoch : 178, Iter:    50] loss: 0.007\n",
      "[Epoch : 179, Iter:    50] loss: 0.007\n",
      "[Epoch : 180, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.70 %\n",
      "Accuracy of   car : 85.60 %\n",
      "Accuracy of  bird : 65.10 %\n",
      "Accuracy of   cat : 55.80 %\n",
      "Accuracy of  deer : 73.10 %\n",
      "Accuracy of   dog : 66.30 %\n",
      "Accuracy of  frog : 84.00 %\n",
      "Accuracy of horse : 80.50 %\n",
      "Accuracy of  ship : 85.20 %\n",
      "Accuracy of truck : 82.70 %\n",
      "76.0\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 82.00 %\n",
      "Accuracy of   car : 85.70 %\n",
      "Accuracy of  bird : 65.10 %\n",
      "Accuracy of   cat : 56.10 %\n",
      "Accuracy of  deer : 73.10 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 83.70 %\n",
      "Accuracy of horse : 80.70 %\n",
      "Accuracy of  ship : 85.30 %\n",
      "Accuracy of truck : 82.90 %\n",
      "76.10999999999999\n",
      "[Epoch : 181, Iter:    50] loss: 0.007\n",
      "[Epoch : 182, Iter:    50] loss: 0.007\n",
      "[Epoch : 183, Iter:    50] loss: 0.007\n",
      "[Epoch : 184, Iter:    50] loss: 0.007\n",
      "[Epoch : 185, Iter:    50] loss: 0.006\n",
      "[Epoch : 186, Iter:    50] loss: 0.007\n",
      "[Epoch : 187, Iter:    50] loss: 0.006\n",
      "[Epoch : 188, Iter:    50] loss: 0.007\n",
      "[Epoch : 189, Iter:    50] loss: 0.007\n",
      "[Epoch : 190, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 82.00 %\n",
      "Accuracy of   car : 85.80 %\n",
      "Accuracy of  bird : 65.10 %\n",
      "Accuracy of   cat : 55.80 %\n",
      "Accuracy of  deer : 72.70 %\n",
      "Accuracy of   dog : 66.80 %\n",
      "Accuracy of  frog : 83.60 %\n",
      "Accuracy of horse : 80.30 %\n",
      "Accuracy of  ship : 85.10 %\n",
      "Accuracy of truck : 83.10 %\n",
      "76.03\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 82.00 %\n",
      "Accuracy of   car : 85.70 %\n",
      "Accuracy of  bird : 65.10 %\n",
      "Accuracy of   cat : 55.90 %\n",
      "Accuracy of  deer : 73.00 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 83.70 %\n",
      "Accuracy of horse : 80.50 %\n",
      "Accuracy of  ship : 85.00 %\n",
      "Accuracy of truck : 82.90 %\n",
      "76.03\n",
      "[Epoch : 191, Iter:    50] loss: 0.006\n",
      "[Epoch : 192, Iter:    50] loss: 0.007\n",
      "[Epoch : 193, Iter:    50] loss: 0.007\n",
      "[Epoch : 194, Iter:    50] loss: 0.006\n",
      "[Epoch : 195, Iter:    50] loss: 0.007\n",
      "[Epoch : 196, Iter:    50] loss: 0.006\n",
      "[Epoch : 197, Iter:    50] loss: 0.007\n",
      "[Epoch : 198, Iter:    50] loss: 0.006\n",
      "[Epoch : 199, Iter:    50] loss: 0.006\n",
      "[Epoch : 200, Iter:    50] loss: 0.007\n",
      "NET_DO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.90 %\n",
      "Accuracy of   car : 85.90 %\n",
      "Accuracy of  bird : 64.80 %\n",
      "Accuracy of   cat : 56.00 %\n",
      "Accuracy of  deer : 72.70 %\n",
      "Accuracy of   dog : 66.70 %\n",
      "Accuracy of  frog : 83.90 %\n",
      "Accuracy of horse : 80.20 %\n",
      "Accuracy of  ship : 85.00 %\n",
      "Accuracy of truck : 83.30 %\n",
      "76.03999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 82.00 %\n",
      "Accuracy of   car : 85.80 %\n",
      "Accuracy of  bird : 65.10 %\n",
      "Accuracy of   cat : 55.90 %\n",
      "Accuracy of  deer : 73.10 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 83.90 %\n",
      "Accuracy of horse : 80.40 %\n",
      "Accuracy of  ship : 85.00 %\n",
      "Accuracy of truck : 82.90 %\n",
      "76.06\n",
      "\n",
      "Net_MCDO\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db0c5e1d0c9049079505cc61dde28083",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=200), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 2.124\n",
      "[Epoch : 2, Iter:    50] loss: 2.003\n",
      "[Epoch : 3, Iter:    50] loss: 1.937\n",
      "[Epoch : 4, Iter:    50] loss: 1.891\n",
      "[Epoch : 5, Iter:    50] loss: 1.850\n",
      "[Epoch : 6, Iter:    50] loss: 1.821\n",
      "[Epoch : 7, Iter:    50] loss: 1.796\n",
      "[Epoch : 8, Iter:    50] loss: 1.776\n",
      "[Epoch : 9, Iter:    50] loss: 1.757\n",
      "[Epoch : 10, Iter:    50] loss: 1.741\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 81.10 %\n",
      "Accuracy of   car : 76.50 %\n",
      "Accuracy of  bird : 46.50 %\n",
      "Accuracy of   cat : 55.50 %\n",
      "Accuracy of  deer : 62.10 %\n",
      "Accuracy of   dog : 62.70 %\n",
      "Accuracy of  frog : 80.70 %\n",
      "Accuracy of horse : 72.80 %\n",
      "Accuracy of  ship : 81.90 %\n",
      "Accuracy of truck : 77.80 %\n",
      "69.76\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 57.50 %\n",
      "Accuracy of   car : 51.40 %\n",
      "Accuracy of  bird : 18.40 %\n",
      "Accuracy of   cat : 42.60 %\n",
      "Accuracy of  deer : 37.90 %\n",
      "Accuracy of   dog : 37.80 %\n",
      "Accuracy of  frog : 51.00 %\n",
      "Accuracy of horse : 55.40 %\n",
      "Accuracy of  ship : 75.70 %\n",
      "Accuracy of truck : 65.20 %\n",
      "49.29\n",
      "[Epoch : 11, Iter:    50] loss: 1.716\n",
      "[Epoch : 12, Iter:    50] loss: 1.700\n",
      "[Epoch : 13, Iter:    50] loss: 1.685\n",
      "[Epoch : 14, Iter:    50] loss: 1.676\n",
      "[Epoch : 15, Iter:    50] loss: 1.667\n",
      "[Epoch : 16, Iter:    50] loss: 1.659\n",
      "[Epoch : 17, Iter:    50] loss: 1.649\n",
      "[Epoch : 18, Iter:    50] loss: 1.645\n",
      "[Epoch : 19, Iter:    50] loss: 1.633\n",
      "[Epoch : 20, Iter:    50] loss: 1.630\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 73.10 %\n",
      "Accuracy of   car : 86.70 %\n",
      "Accuracy of  bird : 69.80 %\n",
      "Accuracy of   cat : 60.30 %\n",
      "Accuracy of  deer : 73.70 %\n",
      "Accuracy of   dog : 61.50 %\n",
      "Accuracy of  frog : 73.90 %\n",
      "Accuracy of horse : 80.80 %\n",
      "Accuracy of  ship : 84.10 %\n",
      "Accuracy of truck : 84.30 %\n",
      "74.82\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 71.30 %\n",
      "Accuracy of   car : 79.60 %\n",
      "Accuracy of  bird : 34.50 %\n",
      "Accuracy of   cat : 48.70 %\n",
      "Accuracy of  deer : 54.60 %\n",
      "Accuracy of   dog : 47.10 %\n",
      "Accuracy of  frog : 79.40 %\n",
      "Accuracy of horse : 72.50 %\n",
      "Accuracy of  ship : 76.00 %\n",
      "Accuracy of truck : 81.10 %\n",
      "64.48\n",
      "[Epoch : 21, Iter:    50] loss: 1.613\n",
      "[Epoch : 22, Iter:    50] loss: 1.597\n",
      "[Epoch : 23, Iter:    50] loss: 1.595\n",
      "[Epoch : 24, Iter:    50] loss: 1.589\n",
      "[Epoch : 25, Iter:    50] loss: 1.583\n",
      "[Epoch : 26, Iter:    50] loss: 1.581\n",
      "[Epoch : 27, Iter:    50] loss: 1.577\n",
      "[Epoch : 28, Iter:    50] loss: 1.575\n",
      "[Epoch : 29, Iter:    50] loss: 1.571\n",
      "[Epoch : 30, Iter:    50] loss: 1.566\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 83.30 %\n",
      "Accuracy of   car : 83.90 %\n",
      "Accuracy of  bird : 71.00 %\n",
      "Accuracy of   cat : 60.80 %\n",
      "Accuracy of  deer : 76.00 %\n",
      "Accuracy of   dog : 58.30 %\n",
      "Accuracy of  frog : 79.10 %\n",
      "Accuracy of horse : 82.50 %\n",
      "Accuracy of  ship : 91.00 %\n",
      "Accuracy of truck : 85.20 %\n",
      "77.11\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 77.60 %\n",
      "Accuracy of   car : 86.00 %\n",
      "Accuracy of  bird : 49.30 %\n",
      "Accuracy of   cat : 54.50 %\n",
      "Accuracy of  deer : 67.00 %\n",
      "Accuracy of   dog : 60.40 %\n",
      "Accuracy of  frog : 85.20 %\n",
      "Accuracy of horse : 80.20 %\n",
      "Accuracy of  ship : 83.40 %\n",
      "Accuracy of truck : 84.80 %\n",
      "72.84\n",
      "[Epoch : 31, Iter:    50] loss: 1.557\n",
      "[Epoch : 32, Iter:    50] loss: 1.551\n",
      "[Epoch : 33, Iter:    50] loss: 1.545\n",
      "[Epoch : 34, Iter:    50] loss: 1.542\n",
      "[Epoch : 35, Iter:    50] loss: 1.539\n",
      "[Epoch : 36, Iter:    50] loss: 1.539\n",
      "[Epoch : 37, Iter:    50] loss: 1.538\n",
      "[Epoch : 38, Iter:    50] loss: 1.535\n",
      "[Epoch : 39, Iter:    50] loss: 1.534\n",
      "[Epoch : 40, Iter:    50] loss: 1.533\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.40 %\n",
      "Accuracy of   car : 89.80 %\n",
      "Accuracy of  bird : 66.30 %\n",
      "Accuracy of   cat : 57.70 %\n",
      "Accuracy of  deer : 73.20 %\n",
      "Accuracy of   dog : 66.80 %\n",
      "Accuracy of  frog : 83.10 %\n",
      "Accuracy of horse : 84.40 %\n",
      "Accuracy of  ship : 89.20 %\n",
      "Accuracy of truck : 84.10 %\n",
      "77.9\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 81.40 %\n",
      "Accuracy of   car : 88.60 %\n",
      "Accuracy of  bird : 58.70 %\n",
      "Accuracy of   cat : 57.20 %\n",
      "Accuracy of  deer : 74.10 %\n",
      "Accuracy of   dog : 66.70 %\n",
      "Accuracy of  frog : 85.70 %\n",
      "Accuracy of horse : 84.10 %\n",
      "Accuracy of  ship : 86.30 %\n",
      "Accuracy of truck : 85.70 %\n",
      "76.85\n",
      "[Epoch : 41, Iter:    50] loss: 1.527\n",
      "[Epoch : 42, Iter:    50] loss: 1.523\n",
      "[Epoch : 43, Iter:    50] loss: 1.520\n",
      "[Epoch : 44, Iter:    50] loss: 1.517\n",
      "[Epoch : 45, Iter:    50] loss: 1.517\n",
      "[Epoch : 46, Iter:    50] loss: 1.517\n",
      "[Epoch : 47, Iter:    50] loss: 1.515\n",
      "[Epoch : 48, Iter:    50] loss: 1.516\n",
      "[Epoch : 49, Iter:    50] loss: 1.514\n",
      "[Epoch : 50, Iter:    50] loss: 1.513\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 83.00 %\n",
      "Accuracy of   car : 86.30 %\n",
      "Accuracy of  bird : 65.30 %\n",
      "Accuracy of   cat : 55.60 %\n",
      "Accuracy of  deer : 81.60 %\n",
      "Accuracy of   dog : 68.00 %\n",
      "Accuracy of  frog : 84.60 %\n",
      "Accuracy of horse : 84.90 %\n",
      "Accuracy of  ship : 88.50 %\n",
      "Accuracy of truck : 87.70 %\n",
      "78.55000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 82.40 %\n",
      "Accuracy of   car : 89.10 %\n",
      "Accuracy of  bird : 63.10 %\n",
      "Accuracy of   cat : 57.50 %\n",
      "Accuracy of  deer : 76.20 %\n",
      "Accuracy of   dog : 69.40 %\n",
      "Accuracy of  frog : 85.10 %\n",
      "Accuracy of horse : 85.10 %\n",
      "Accuracy of  ship : 87.90 %\n",
      "Accuracy of truck : 85.80 %\n",
      "78.16\n",
      "[Epoch : 51, Iter:    50] loss: 1.510\n",
      "[Epoch : 52, Iter:    50] loss: 1.507\n",
      "[Epoch : 53, Iter:    50] loss: 1.505\n",
      "[Epoch : 54, Iter:    50] loss: 1.505\n",
      "[Epoch : 55, Iter:    50] loss: 1.505\n",
      "[Epoch : 56, Iter:    50] loss: 1.504\n",
      "[Epoch : 57, Iter:    50] loss: 1.503\n",
      "[Epoch : 58, Iter:    50] loss: 1.504\n",
      "[Epoch : 59, Iter:    50] loss: 1.502\n",
      "[Epoch : 60, Iter:    50] loss: 1.502\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 88.00 %\n",
      "Accuracy of   car : 89.60 %\n",
      "Accuracy of  bird : 71.60 %\n",
      "Accuracy of   cat : 57.40 %\n",
      "Accuracy of  deer : 72.70 %\n",
      "Accuracy of   dog : 65.10 %\n",
      "Accuracy of  frog : 84.50 %\n",
      "Accuracy of horse : 85.80 %\n",
      "Accuracy of  ship : 87.70 %\n",
      "Accuracy of truck : 84.70 %\n",
      "78.71000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 83.00 %\n",
      "Accuracy of   car : 89.60 %\n",
      "Accuracy of  bird : 66.10 %\n",
      "Accuracy of   cat : 60.20 %\n",
      "Accuracy of  deer : 76.50 %\n",
      "Accuracy of   dog : 70.40 %\n",
      "Accuracy of  frog : 84.00 %\n",
      "Accuracy of horse : 85.50 %\n",
      "Accuracy of  ship : 88.70 %\n",
      "Accuracy of truck : 85.80 %\n",
      "78.97999999999999\n",
      "[Epoch : 61, Iter:    50] loss: 1.500\n",
      "[Epoch : 62, Iter:    50] loss: 1.498\n",
      "[Epoch : 63, Iter:    50] loss: 1.497\n",
      "[Epoch : 64, Iter:    50] loss: 1.497\n",
      "[Epoch : 65, Iter:    50] loss: 1.496\n",
      "[Epoch : 66, Iter:    50] loss: 1.496\n",
      "[Epoch : 67, Iter:    50] loss: 1.496\n",
      "[Epoch : 68, Iter:    50] loss: 1.495\n",
      "[Epoch : 69, Iter:    50] loss: 1.495\n",
      "[Epoch : 70, Iter:    50] loss: 1.496\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.20 %\n",
      "Accuracy of   car : 88.30 %\n",
      "Accuracy of  bird : 65.90 %\n",
      "Accuracy of   cat : 61.70 %\n",
      "Accuracy of  deer : 78.20 %\n",
      "Accuracy of   dog : 68.00 %\n",
      "Accuracy of  frog : 86.10 %\n",
      "Accuracy of horse : 85.40 %\n",
      "Accuracy of  ship : 87.70 %\n",
      "Accuracy of truck : 86.30 %\n",
      "79.17999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.60 %\n",
      "Accuracy of   car : 89.30 %\n",
      "Accuracy of  bird : 68.20 %\n",
      "Accuracy of   cat : 60.20 %\n",
      "Accuracy of  deer : 78.10 %\n",
      "Accuracy of   dog : 70.50 %\n",
      "Accuracy of  frog : 85.40 %\n",
      "Accuracy of horse : 86.20 %\n",
      "Accuracy of  ship : 88.60 %\n",
      "Accuracy of truck : 85.70 %\n",
      "79.68\n",
      "[Epoch : 71, Iter:    50] loss: 1.494\n",
      "[Epoch : 72, Iter:    50] loss: 1.493\n",
      "[Epoch : 73, Iter:    50] loss: 1.493\n",
      "[Epoch : 74, Iter:    50] loss: 1.492\n",
      "[Epoch : 75, Iter:    50] loss: 1.492\n",
      "[Epoch : 76, Iter:    50] loss: 1.491\n",
      "[Epoch : 77, Iter:    50] loss: 1.491\n",
      "[Epoch : 78, Iter:    50] loss: 1.491\n",
      "[Epoch : 79, Iter:    50] loss: 1.491\n",
      "[Epoch : 80, Iter:    50] loss: 1.491\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.10 %\n",
      "Accuracy of   car : 86.70 %\n",
      "Accuracy of  bird : 74.40 %\n",
      "Accuracy of   cat : 61.50 %\n",
      "Accuracy of  deer : 76.90 %\n",
      "Accuracy of   dog : 64.80 %\n",
      "Accuracy of  frog : 84.60 %\n",
      "Accuracy of horse : 86.00 %\n",
      "Accuracy of  ship : 89.50 %\n",
      "Accuracy of truck : 84.90 %\n",
      "79.34\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 85.00 %\n",
      "Accuracy of   car : 89.40 %\n",
      "Accuracy of  bird : 69.90 %\n",
      "Accuracy of   cat : 61.40 %\n",
      "Accuracy of  deer : 77.80 %\n",
      "Accuracy of   dog : 69.90 %\n",
      "Accuracy of  frog : 83.80 %\n",
      "Accuracy of horse : 86.00 %\n",
      "Accuracy of  ship : 88.80 %\n",
      "Accuracy of truck : 85.90 %\n",
      "79.78999999999999\n",
      "[Epoch : 81, Iter:    50] loss: 1.491\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 82, Iter:    50] loss: 1.489\n",
      "[Epoch : 83, Iter:    50] loss: 1.489\n",
      "[Epoch : 84, Iter:    50] loss: 1.489\n",
      "[Epoch : 85, Iter:    50] loss: 1.489\n",
      "[Epoch : 86, Iter:    50] loss: 1.488\n",
      "[Epoch : 87, Iter:    50] loss: 1.488\n",
      "[Epoch : 88, Iter:    50] loss: 1.488\n",
      "[Epoch : 89, Iter:    50] loss: 1.488\n",
      "[Epoch : 90, Iter:    50] loss: 1.488\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 80.90 %\n",
      "Accuracy of   car : 88.80 %\n",
      "Accuracy of  bird : 69.60 %\n",
      "Accuracy of   cat : 63.10 %\n",
      "Accuracy of  deer : 78.60 %\n",
      "Accuracy of   dog : 68.90 %\n",
      "Accuracy of  frog : 85.10 %\n",
      "Accuracy of horse : 84.60 %\n",
      "Accuracy of  ship : 88.40 %\n",
      "Accuracy of truck : 84.00 %\n",
      "79.19999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.70 %\n",
      "Accuracy of   car : 90.30 %\n",
      "Accuracy of  bird : 70.00 %\n",
      "Accuracy of   cat : 60.90 %\n",
      "Accuracy of  deer : 77.10 %\n",
      "Accuracy of   dog : 69.80 %\n",
      "Accuracy of  frog : 84.10 %\n",
      "Accuracy of horse : 85.60 %\n",
      "Accuracy of  ship : 88.70 %\n",
      "Accuracy of truck : 85.90 %\n",
      "79.71000000000001\n",
      "[Epoch : 91, Iter:    50] loss: 1.487\n",
      "[Epoch : 92, Iter:    50] loss: 1.486\n",
      "[Epoch : 93, Iter:    50] loss: 1.486\n",
      "[Epoch : 94, Iter:    50] loss: 1.486\n",
      "[Epoch : 95, Iter:    50] loss: 1.486\n",
      "[Epoch : 96, Iter:    50] loss: 1.486\n",
      "[Epoch : 97, Iter:    50] loss: 1.486\n",
      "[Epoch : 98, Iter:    50] loss: 1.486\n",
      "[Epoch : 99, Iter:    50] loss: 1.486\n",
      "[Epoch : 100, Iter:    50] loss: 1.485\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 83.40 %\n",
      "Accuracy of   car : 89.20 %\n",
      "Accuracy of  bird : 73.10 %\n",
      "Accuracy of   cat : 61.50 %\n",
      "Accuracy of  deer : 73.80 %\n",
      "Accuracy of   dog : 68.10 %\n",
      "Accuracy of  frog : 85.50 %\n",
      "Accuracy of horse : 83.80 %\n",
      "Accuracy of  ship : 88.90 %\n",
      "Accuracy of truck : 84.40 %\n",
      "79.17\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 85.60 %\n",
      "Accuracy of   car : 89.40 %\n",
      "Accuracy of  bird : 71.10 %\n",
      "Accuracy of   cat : 61.50 %\n",
      "Accuracy of  deer : 76.80 %\n",
      "Accuracy of   dog : 70.10 %\n",
      "Accuracy of  frog : 84.30 %\n",
      "Accuracy of horse : 85.20 %\n",
      "Accuracy of  ship : 88.60 %\n",
      "Accuracy of truck : 85.90 %\n",
      "79.85\n",
      "[Epoch : 101, Iter:    50] loss: 1.484\n",
      "[Epoch : 102, Iter:    50] loss: 1.484\n",
      "[Epoch : 103, Iter:    50] loss: 1.484\n",
      "[Epoch : 104, Iter:    50] loss: 1.484\n",
      "[Epoch : 105, Iter:    50] loss: 1.484\n",
      "[Epoch : 106, Iter:    50] loss: 1.484\n",
      "[Epoch : 107, Iter:    50] loss: 1.484\n",
      "[Epoch : 108, Iter:    50] loss: 1.484\n",
      "[Epoch : 109, Iter:    50] loss: 1.484\n",
      "[Epoch : 110, Iter:    50] loss: 1.484\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 85.20 %\n",
      "Accuracy of   car : 88.70 %\n",
      "Accuracy of  bird : 70.10 %\n",
      "Accuracy of   cat : 58.80 %\n",
      "Accuracy of  deer : 78.40 %\n",
      "Accuracy of   dog : 69.00 %\n",
      "Accuracy of  frog : 85.80 %\n",
      "Accuracy of horse : 84.60 %\n",
      "Accuracy of  ship : 87.80 %\n",
      "Accuracy of truck : 85.50 %\n",
      "79.38999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.70 %\n",
      "Accuracy of   car : 89.60 %\n",
      "Accuracy of  bird : 70.30 %\n",
      "Accuracy of   cat : 60.90 %\n",
      "Accuracy of  deer : 77.20 %\n",
      "Accuracy of   dog : 69.80 %\n",
      "Accuracy of  frog : 84.50 %\n",
      "Accuracy of horse : 85.20 %\n",
      "Accuracy of  ship : 88.10 %\n",
      "Accuracy of truck : 86.00 %\n",
      "79.63000000000001\n",
      "[Epoch : 111, Iter:    50] loss: 1.484\n",
      "[Epoch : 112, Iter:    50] loss: 1.483\n",
      "[Epoch : 113, Iter:    50] loss: 1.483\n",
      "[Epoch : 114, Iter:    50] loss: 1.483\n",
      "[Epoch : 115, Iter:    50] loss: 1.483\n",
      "[Epoch : 116, Iter:    50] loss: 1.483\n",
      "[Epoch : 117, Iter:    50] loss: 1.483\n",
      "[Epoch : 118, Iter:    50] loss: 1.483\n",
      "[Epoch : 119, Iter:    50] loss: 1.483\n",
      "[Epoch : 120, Iter:    50] loss: 1.483\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 85.70 %\n",
      "Accuracy of   car : 88.20 %\n",
      "Accuracy of  bird : 71.10 %\n",
      "Accuracy of   cat : 59.90 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 68.80 %\n",
      "Accuracy of  frog : 86.80 %\n",
      "Accuracy of horse : 84.90 %\n",
      "Accuracy of  ship : 89.40 %\n",
      "Accuracy of truck : 84.50 %\n",
      "79.72999999999999\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.30 %\n",
      "Accuracy of   car : 88.90 %\n",
      "Accuracy of  bird : 70.40 %\n",
      "Accuracy of   cat : 60.40 %\n",
      "Accuracy of  deer : 77.60 %\n",
      "Accuracy of   dog : 69.50 %\n",
      "Accuracy of  frog : 84.90 %\n",
      "Accuracy of horse : 85.00 %\n",
      "Accuracy of  ship : 89.10 %\n",
      "Accuracy of truck : 85.80 %\n",
      "79.59\n",
      "[Epoch : 121, Iter:    50] loss: 1.482\n",
      "[Epoch : 122, Iter:    50] loss: 1.482\n",
      "[Epoch : 123, Iter:    50] loss: 1.482\n",
      "[Epoch : 124, Iter:    50] loss: 1.482\n",
      "[Epoch : 125, Iter:    50] loss: 1.482\n",
      "[Epoch : 126, Iter:    50] loss: 1.482\n",
      "[Epoch : 127, Iter:    50] loss: 1.482\n",
      "[Epoch : 128, Iter:    50] loss: 1.482\n",
      "[Epoch : 129, Iter:    50] loss: 1.482\n",
      "[Epoch : 130, Iter:    50] loss: 1.482\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.70 %\n",
      "Accuracy of   car : 89.30 %\n",
      "Accuracy of  bird : 69.40 %\n",
      "Accuracy of   cat : 60.60 %\n",
      "Accuracy of  deer : 77.20 %\n",
      "Accuracy of   dog : 66.80 %\n",
      "Accuracy of  frog : 86.30 %\n",
      "Accuracy of horse : 84.80 %\n",
      "Accuracy of  ship : 88.30 %\n",
      "Accuracy of truck : 85.90 %\n",
      "79.33\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 86.00 %\n",
      "Accuracy of   car : 88.80 %\n",
      "Accuracy of  bird : 71.70 %\n",
      "Accuracy of   cat : 61.40 %\n",
      "Accuracy of  deer : 77.20 %\n",
      "Accuracy of   dog : 68.70 %\n",
      "Accuracy of  frog : 85.40 %\n",
      "Accuracy of horse : 85.70 %\n",
      "Accuracy of  ship : 88.60 %\n",
      "Accuracy of truck : 85.60 %\n",
      "79.91\n",
      "[Epoch : 131, Iter:    50] loss: 1.482\n",
      "[Epoch : 132, Iter:    50] loss: 1.482\n",
      "[Epoch : 133, Iter:    50] loss: 1.481\n",
      "[Epoch : 134, Iter:    50] loss: 1.482\n",
      "[Epoch : 135, Iter:    50] loss: 1.481\n",
      "[Epoch : 136, Iter:    50] loss: 1.481\n",
      "[Epoch : 137, Iter:    50] loss: 1.482\n",
      "[Epoch : 138, Iter:    50] loss: 1.481\n",
      "[Epoch : 139, Iter:    50] loss: 1.481\n",
      "[Epoch : 140, Iter:    50] loss: 1.482\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 85.70 %\n",
      "Accuracy of   car : 88.30 %\n",
      "Accuracy of  bird : 70.60 %\n",
      "Accuracy of   cat : 62.70 %\n",
      "Accuracy of  deer : 77.40 %\n",
      "Accuracy of   dog : 67.10 %\n",
      "Accuracy of  frog : 86.50 %\n",
      "Accuracy of horse : 85.30 %\n",
      "Accuracy of  ship : 87.60 %\n",
      "Accuracy of truck : 85.30 %\n",
      "79.65\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 85.10 %\n",
      "Accuracy of   car : 89.30 %\n",
      "Accuracy of  bird : 71.00 %\n",
      "Accuracy of   cat : 60.90 %\n",
      "Accuracy of  deer : 77.30 %\n",
      "Accuracy of   dog : 69.90 %\n",
      "Accuracy of  frog : 85.60 %\n",
      "Accuracy of horse : 85.40 %\n",
      "Accuracy of  ship : 88.90 %\n",
      "Accuracy of truck : 86.10 %\n",
      "79.95\n",
      "[Epoch : 141, Iter:    50] loss: 1.481\n",
      "[Epoch : 142, Iter:    50] loss: 1.481\n",
      "[Epoch : 143, Iter:    50] loss: 1.481\n",
      "[Epoch : 144, Iter:    50] loss: 1.481\n",
      "[Epoch : 145, Iter:    50] loss: 1.481\n",
      "[Epoch : 146, Iter:    50] loss: 1.481\n",
      "[Epoch : 147, Iter:    50] loss: 1.481\n",
      "[Epoch : 148, Iter:    50] loss: 1.481\n",
      "[Epoch : 149, Iter:    50] loss: 1.481\n",
      "[Epoch : 150, Iter:    50] loss: 1.481\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 85.20 %\n",
      "Accuracy of   car : 88.70 %\n",
      "Accuracy of  bird : 70.70 %\n",
      "Accuracy of   cat : 62.00 %\n",
      "Accuracy of  deer : 77.50 %\n",
      "Accuracy of   dog : 68.50 %\n",
      "Accuracy of  frog : 86.40 %\n",
      "Accuracy of horse : 85.00 %\n",
      "Accuracy of  ship : 88.40 %\n",
      "Accuracy of truck : 84.60 %\n",
      "79.7\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.50 %\n",
      "Accuracy of   car : 89.20 %\n",
      "Accuracy of  bird : 71.00 %\n",
      "Accuracy of   cat : 60.50 %\n",
      "Accuracy of  deer : 78.30 %\n",
      "Accuracy of   dog : 68.60 %\n",
      "Accuracy of  frog : 86.30 %\n",
      "Accuracy of horse : 85.00 %\n",
      "Accuracy of  ship : 88.30 %\n",
      "Accuracy of truck : 85.00 %\n",
      "79.66999999999999\n",
      "[Epoch : 151, Iter:    50] loss: 1.481\n",
      "[Epoch : 152, Iter:    50] loss: 1.481\n",
      "[Epoch : 153, Iter:    50] loss: 1.480\n",
      "[Epoch : 154, Iter:    50] loss: 1.481\n",
      "[Epoch : 155, Iter:    50] loss: 1.481\n",
      "[Epoch : 156, Iter:    50] loss: 1.481\n",
      "[Epoch : 157, Iter:    50] loss: 1.481\n",
      "[Epoch : 158, Iter:    50] loss: 1.481\n",
      "[Epoch : 159, Iter:    50] loss: 1.480\n",
      "[Epoch : 160, Iter:    50] loss: 1.481\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.70 %\n",
      "Accuracy of   car : 89.20 %\n",
      "Accuracy of  bird : 68.20 %\n",
      "Accuracy of   cat : 62.20 %\n",
      "Accuracy of  deer : 79.20 %\n",
      "Accuracy of   dog : 67.60 %\n",
      "Accuracy of  frog : 86.50 %\n",
      "Accuracy of horse : 84.70 %\n",
      "Accuracy of  ship : 88.80 %\n",
      "Accuracy of truck : 85.80 %\n",
      "79.68999999999998\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.40 %\n",
      "Accuracy of   car : 88.90 %\n",
      "Accuracy of  bird : 71.00 %\n",
      "Accuracy of   cat : 60.70 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 68.10 %\n",
      "Accuracy of  frog : 85.50 %\n",
      "Accuracy of horse : 84.70 %\n",
      "Accuracy of  ship : 88.90 %\n",
      "Accuracy of truck : 85.70 %\n",
      "79.59\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 161, Iter:    50] loss: 1.480\n",
      "[Epoch : 162, Iter:    50] loss: 1.480\n",
      "[Epoch : 163, Iter:    50] loss: 1.481\n",
      "[Epoch : 164, Iter:    50] loss: 1.480\n",
      "[Epoch : 165, Iter:    50] loss: 1.480\n",
      "[Epoch : 166, Iter:    50] loss: 1.481\n",
      "[Epoch : 167, Iter:    50] loss: 1.480\n",
      "[Epoch : 168, Iter:    50] loss: 1.480\n",
      "[Epoch : 169, Iter:    50] loss: 1.480\n",
      "[Epoch : 170, Iter:    50] loss: 1.480\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.30 %\n",
      "Accuracy of   car : 89.90 %\n",
      "Accuracy of  bird : 71.50 %\n",
      "Accuracy of   cat : 60.90 %\n",
      "Accuracy of  deer : 77.80 %\n",
      "Accuracy of   dog : 68.40 %\n",
      "Accuracy of  frog : 87.00 %\n",
      "Accuracy of horse : 84.60 %\n",
      "Accuracy of  ship : 88.50 %\n",
      "Accuracy of truck : 86.00 %\n",
      "79.89\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 85.30 %\n",
      "Accuracy of   car : 89.20 %\n",
      "Accuracy of  bird : 70.60 %\n",
      "Accuracy of   cat : 61.60 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 68.40 %\n",
      "Accuracy of  frog : 85.50 %\n",
      "Accuracy of horse : 85.20 %\n",
      "Accuracy of  ship : 88.50 %\n",
      "Accuracy of truck : 85.50 %\n",
      "79.78\n",
      "[Epoch : 171, Iter:    50] loss: 1.480\n",
      "[Epoch : 172, Iter:    50] loss: 1.481\n",
      "[Epoch : 173, Iter:    50] loss: 1.480\n",
      "[Epoch : 174, Iter:    50] loss: 1.480\n",
      "[Epoch : 175, Iter:    50] loss: 1.480\n",
      "[Epoch : 176, Iter:    50] loss: 1.480\n",
      "[Epoch : 177, Iter:    50] loss: 1.480\n",
      "[Epoch : 178, Iter:    50] loss: 1.480\n",
      "[Epoch : 179, Iter:    50] loss: 1.480\n",
      "[Epoch : 180, Iter:    50] loss: 1.480\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 85.10 %\n",
      "Accuracy of   car : 88.30 %\n",
      "Accuracy of  bird : 72.00 %\n",
      "Accuracy of   cat : 62.30 %\n",
      "Accuracy of  deer : 77.80 %\n",
      "Accuracy of   dog : 67.50 %\n",
      "Accuracy of  frog : 86.20 %\n",
      "Accuracy of horse : 85.30 %\n",
      "Accuracy of  ship : 88.10 %\n",
      "Accuracy of truck : 86.20 %\n",
      "79.88000000000001\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.50 %\n",
      "Accuracy of   car : 88.90 %\n",
      "Accuracy of  bird : 70.80 %\n",
      "Accuracy of   cat : 60.90 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 69.40 %\n",
      "Accuracy of  frog : 86.30 %\n",
      "Accuracy of horse : 84.80 %\n",
      "Accuracy of  ship : 87.80 %\n",
      "Accuracy of truck : 86.10 %\n",
      "79.75\n",
      "[Epoch : 181, Iter:    50] loss: 1.480\n",
      "[Epoch : 182, Iter:    50] loss: 1.480\n",
      "[Epoch : 183, Iter:    50] loss: 1.480\n",
      "[Epoch : 184, Iter:    50] loss: 1.480\n",
      "[Epoch : 185, Iter:    50] loss: 1.480\n",
      "[Epoch : 186, Iter:    50] loss: 1.480\n",
      "[Epoch : 187, Iter:    50] loss: 1.480\n",
      "[Epoch : 188, Iter:    50] loss: 1.480\n",
      "[Epoch : 189, Iter:    50] loss: 1.480\n",
      "[Epoch : 190, Iter:    50] loss: 1.480\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.70 %\n",
      "Accuracy of   car : 89.00 %\n",
      "Accuracy of  bird : 70.20 %\n",
      "Accuracy of   cat : 61.60 %\n",
      "Accuracy of  deer : 78.40 %\n",
      "Accuracy of   dog : 68.90 %\n",
      "Accuracy of  frog : 86.50 %\n",
      "Accuracy of horse : 83.90 %\n",
      "Accuracy of  ship : 89.00 %\n",
      "Accuracy of truck : 85.10 %\n",
      "79.73\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.60 %\n",
      "Accuracy of   car : 89.30 %\n",
      "Accuracy of  bird : 70.80 %\n",
      "Accuracy of   cat : 61.60 %\n",
      "Accuracy of  deer : 78.10 %\n",
      "Accuracy of   dog : 68.50 %\n",
      "Accuracy of  frog : 87.00 %\n",
      "Accuracy of horse : 84.30 %\n",
      "Accuracy of  ship : 88.60 %\n",
      "Accuracy of truck : 86.20 %\n",
      "79.9\n",
      "[Epoch : 191, Iter:    50] loss: 1.480\n",
      "[Epoch : 192, Iter:    50] loss: 1.480\n",
      "[Epoch : 193, Iter:    50] loss: 1.480\n",
      "[Epoch : 194, Iter:    50] loss: 1.480\n",
      "[Epoch : 195, Iter:    50] loss: 1.480\n",
      "[Epoch : 196, Iter:    50] loss: 1.480\n",
      "[Epoch : 197, Iter:    50] loss: 1.480\n",
      "[Epoch : 198, Iter:    50] loss: 1.480\n",
      "[Epoch : 199, Iter:    50] loss: 1.480\n",
      "[Epoch : 200, Iter:    50] loss: 1.480\n",
      "NET_MCDO TEST\n",
      "Train net test\n",
      "Start test\n",
      "Accuracy of plane : 84.80 %\n",
      "Accuracy of   car : 88.50 %\n",
      "Accuracy of  bird : 71.50 %\n",
      "Accuracy of   cat : 61.80 %\n",
      "Accuracy of  deer : 78.50 %\n",
      "Accuracy of   dog : 67.90 %\n",
      "Accuracy of  frog : 85.30 %\n",
      "Accuracy of horse : 85.10 %\n",
      "Accuracy of  ship : 88.70 %\n",
      "Accuracy of truck : 85.10 %\n",
      "79.72\n",
      "Test net test\n",
      "Start test\n",
      "Accuracy of plane : 84.50 %\n",
      "Accuracy of   car : 88.70 %\n",
      "Accuracy of  bird : 71.40 %\n",
      "Accuracy of   cat : 60.70 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 69.30 %\n",
      "Accuracy of  frog : 86.00 %\n",
      "Accuracy of horse : 84.20 %\n",
      "Accuracy of  ship : 87.50 %\n",
      "Accuracy of truck : 85.90 %\n",
      "79.61999999999999\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if is_train:\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(net_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEUCAYAAABkhkJAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXmcHkWdx/+u7n7uZ45n7skxOclJCCHhFhEMRDCYiKxAXAERgYCKPxZWdJUoCrug+5N1+fnaXRdXEUVEzgQ0IIcCcsmRa5KQMDOZSTKZ+3ju5+nu+v3Rzzwzk/OZyZxJvfPqVNW3q6u//TzP1KerurpKSCklCoVCoVCMMtpoO6BQKBQKBShBUigUCsUYQQmSQqFQKMYESpAUCoVCMSZQgqRQKBSKMYESJIVCoVCMCZQgKRQKhWJMoARJoVAoFGMCJUgKhUKhGBMoQVIoFArFmEAJkkKhUCjGBEqQFAqFQjEmUIKkUCgUijGBMdoODAUdHVFse3CTlhcXB2lriwyxR8OD8nV4UL4OD8rX4WE8+KppglAoMODjjglBsm05aEHqOX68oHwdHpSvw4PydXgYT74OBNVlp1AoFIoxgRIkhUKhUIwJlCApFAqFYkwwYoL08ssvs3LlSlasWMFnPvMZnn/+eQBqa2u5/PLLWbZsGZdffjl1dXUj5ZJCoVAoxhAjMqhBSsk///M/85vf/IZZs2axbds2rrzySpYuXcqaNWtYtWoVK1as4Omnn+bOO+/koYceGgm3FAqFQjGGGLEWkqZphMNhAMLhMGVlZXR0dFBdXc3y5csBWL58OdXV1bS3t4+UWwqFYgSRUmJbFmY6RTqVRMpjc7SYYnCMSAtJCMH999/PTTfdhN/vJxqN8j//8z80NjZSXl6OrusA6LpOWVkZjY2NFBUV5Vx+cXHwqPwrLc07quNHEuXrwJBSIoQ4Yr6h8NW2LFKpBOlkglQiTiqZIJWMO+lkHN1w4Qvk4w/m4Q3k4fPnoRsD/xMcqc9VSkkiFiXc2Uq4s41wR5sTdrYS7e7Etkxs23Y2y0LaNrZtYds2MhPado/dse0vQLphEMgLESwIEcgPESgoJJjvxIP5IQIFIQJ5hYP6nAbK4T5XW9pEUzHCqSiRZJTuZIRIqjcMJ6PE0nEABIAQCEAg9os7Yf94T36BJgRew4Pf5cPn8uIzvPg0N25bYNhg2ILWFgvNkljJZJ/fWpxkIk466fzu3F5f7+dYECKQX0iwoAh/XgG6Pnbf9hkRz0zT5L//+7/52c9+xuLFi3n33Xf5xje+wX333Tck5be1RQY9Lr+0NI+WlvCQ+DHcHE++Rrvb6WzZg5VOYWY2y+yJJ7HMNGY66djTKczMvp78PTbbMhGahq670A0DTXeh6waa0ZM28Pm8WLZA111ouuHYDSefrjt5LDNNOp3ETCZIpxKYKSdMp5KZeBLLTA34Ol0eHx5fEI8/iNcXxOPPy6QDeH15ePxBPL4gXn8ebl+AiorQgD5Xy7ZoT3TSlminLd5Oa6KdhJmg2FdEma+EkPTiTkkSkS5i3e1Eu9uJdXcQDbcT627HTPe/Jt1w4c8vwh8sxOX1IzQNTdMQQkdoAqHpaEJDaBr+gJdk0kJk0k4+DaHpCCFIS5N4LEwi0kU02k173YekomFsyzzgOnSPF8MfQPf50f0+NK8P4fM6m8uF0HU0TQfdQDM0hGag6bpzLs25IekjC/1FAvAFDPZ1tBNJR4mmY0STEeKxMMlElFQiipVIols2LgsMU/YLXabEZQt0S4IQSOdk2Jnz9KSlIBMXyMw9Uq/NCSUgLBvdkuiWxLBBy6FqszWBNHQwdIRhIEwLEimEbffLJwE8LvB4EF4PeL3gc4PXi/B6EF4v+LwIw+DUikVMyZ985JMfBE0Tg2oojIggbd26lebmZhYvXgzA4sWL8fl8eDwempqasCwLXdexLIvm5mYqKytHwi3FGCKViNHcsIN99dtprv+QcEfzIXIKDJcbw+VGd7kxjEzo8uAP+tBdHgzD5YQuN7puYNsWlmVim2ks08Sy0tiWiWWa2JZjSyQS2KaJZZlYZjprd1oBFpqu43J7MdxeXJnNG8gnL9RrM9yefmH//B4syyQZi5CIhUnGIyTjERIxJ0zGIkQ6W2lrrCMZjyKlfdCrN1xuEAKtRwCE7lRqAmwkNhITG0tapLEwpZWt9KRw7sp1NOJJi5a0PLCyc7lwBYP484qYNHEaBYXlBPKLCOQX4c8P4fEFc2px2tJGD1rs2LOb9kQH7YnOTOjEO+IdpOy0kzmY2QCkjmHpuNMST2Zzp8GTTuNOd+Bpb8fdJHFbR/5NZX0RmU3rCUU2LTNpJLgsicsEw4KQLQkdqkCh4fJ6cXsDeAIBvN4gHl8Aw+0FKZHSRkqJtO2DxHvSsjdtZ2zSBikx3B50t8cRWsNAGgZSF9iGhqkJjKBBVyJJStgkNZs4KRJ2mriZIGElSJgJrExZWtrGlbYwUhaulO1saQtXKokrHMfdbmOYzo+gRxABLA02L2xnynnX5P5BDwEjIkgVFRXs27ePmpoapk+fzkcffURbWxtTpkxh7ty5rFu3jhUrVrBu3Trmzp07oO46xfjEMtO07q2hqf5DmnZtp6O5ASklhstD6aSZzFh4NiWV03B5fI6wGI4IabqRU4UIYNomSSuFW3fj0g79Uz9Sa05KGyGG6HFrUfkRs0hpk0rEiUY76exuoTvSTjjSQSzajWnFCcfCxM0EyXSCpBkDKRFSOvf6EtzChUdz4dH85AkDQ+i4hI7zT6BpOu6KILbPQ9KtETFM2kWCfXY3Lal2TBkH9gB78MV8lFNKuVlKWaKEMn8p5f5SiryFhFORA4SmJ96R7MLeT1SDrgBF3kIqA2XML55NyFuIV/egC6dVpWs6utDQhe5smoYm+tgy+zWhI2ybdDxGOhrBTCWyNxiWmca2e282em4yem9A+mx99rldBobLj9cXxO0L4PH6cXv9uL0BR3h8PXE/hsuT829wOBjqnhLLTBOPdGW2TmKRLuLRLiafsHDIzpErQo7QU8VnnnmGn//859kv8utf/zpLly7lo48+4o477qC7u5v8/Hzuvfdepk+fPqCyVZfd2GN/X23bprNlN027ttNU/yGte2qwrDRC0yiunEp51WzKq2ZTXDEFLfNMMW4miJvxTHioeF9bf3va7u360YWO1/Dg1b2Z0IPH8ODTvRQG85Bpgdfw4tU9++VzQl043U0aGpoQTreUEIg+aS2b7on35gNIW2ki6SjhdOa5QypCJB11bKkIkR57OkokFSFhJQ/62QZcfoq9RRT7iijxFlHsC2XTRZ5CXLpr0N+bLW3aEx00xVqyW3OsleZYC53JrkMeJxAUePIp8oYo8hZS5A0xpaQSl+mj2FtIyBvCo7sH7ddwM57/tsYig+2yGzFBGk6UII09SkqC1O6opal+O03122lu2EEqEQOgoGQC5VWzKK+aTemkGUhdpzG6j72RfeyJNjphpJFIOnrYc7g0A6/hxW/49gu9+AwfPsOLW3eTstIkrSQJM0HCSpIwk5nQ6eJI22mi6Tgpa+DPgHJBIBBCHNBi6EETGnmuAEF3kDxXkKA7QNAVIM8dJLifffqESqKdBz5jGQkSZpLmuCNQHYlO8tzBjACFCHkK0DW9X/7x9HtVvg4tY/oZkuL4wDLTNDfsYM9Hm2iu30a4sw0Af16IiTMWUFY1C6O0jDYZZk+kkXcj77HnvWdpibUhM73XLs3FhEAFC0rmUe4vdUYbZcRlf+E5XDfcQOj5A7elnRGuZB/hSpAwk9jSwpYSW9rYOP39trQdG84zAFvaSGTWns2TsXl0dx+hcQQmzxXEZ3hz7gLyu3xEGZ3KyGt4qMqbRFXepFE5v+LYRwmS4qhIJWI01laz56ONNNZuxUwnMVxuJsyYS+GcBcRCfpoJsz3axJ7mp0k1Oq0QgaDEV8SEYCVLyk5mQrCSicEKSnzFaEP1vGaAaELLiJ9vVM6vUBzvKEFSHICUknRm1E7PM5mYmSBhxomZCaLd7UR312PubUS2dyKkxHLpRIp8tBf6aPXbxOV2SACNEDD8TAhWcGblqUwMVjAhUElloByv4RntS1UoFGMIJUjHOVJK2hLt7OioYUdnDR911tKR7MKSVt9MBBKS4i6b4i6LvLjTvRb3akQrA6RKC9AKC/C5/UzUvcx0eZlYVEo+RUwMVlDgzh/VUUkKhWJ8oATpOENKSXOshR2djgDt7KzNjp4KugLMLJzGKeUL8Wpu9M4w5t59xPY0kI44zy0KK6qYeMoCJs9cSEFxxSHPMx4evCoUirGFEqRjHCkljdEmdvYRoO6UIxR57iAnFE7nhMLpzCycToW/jKZd26jf/h57azaTSsTQdIPyqtlMPHMBE6efiDeQP8pXpFAojlWUIB1j2NJmb2RfRnwcAeoZPl3oKWBWaEZWhMr8pQghkNJmz85NvPDmr+hs2YPb46dy+nwmzlhAxdS5uNzqWY9CoRh+lCAdA3Qlu9nctpUtrdvY2VlL1HTe9ynyhphfPIeZGQEq8RX1e5YjpU3DhxvY8uaf6GrdS7CwlNOWfYEpc5ZkX05VKBSKkUIJ0jhESklDZA+bWreyuXUr9eHdAIQ8hZxUOj/bBVfsO/hsXFLa7N6xgS1vrqerdS95oTJO/9Q/UjVnsTNBpUKhUIwCSpDGCSkrxd/3bOT1mvfY3LqVrlQ3AsHU/Coumf4pFpTMZUKg4rCj2WzbZveOD9jy5nq62xodIbroi1TNXoymqdXsFQrF6KIEaQzTmezKtIKq2d6xk7Rt4tHdzC2azYKSucwvnkOe+8jTc9i2TcP296h+az3d7U3kF5VzxsVXM3nWIiVECoVizKAEaQxhS5uG8J6sCDVE9gJQ7C3i7Amn87EZiykRFTlPmWPbFvXbHCEKdzSTX1zJmZ++hsmzTh662asVCoViiFCCNAboSnbzbO3zbGrdSncqjEAwvWAKK2ZcxIKSeVT4yxBC5Pxuj21b7Nr2LtVvrifS2UJByQTOWv4lJp2wUAmRQqEYsyhBGmX2Rvbxsw2/IJKOsqBkLgtK5jGvaDZBd2DAZdmWxa5tf6f6reeJdLZQWDqRsy/5MhNnLlBCpFAoxjxKkEaRbe07+PmmX+PRXfzT4puYnDdx0GUl4xFe/N39hDuaKSybxMc+cx0TZixQU/YoFIpxgxKkUeKNve/w2+2PU+Ev46aF1xLyFh5Vee+99DjRrrZMi+gkJUQKhWLcoQRphJFSsq72ef5U9yJzQidw3YIv4jO8R1Xmnp0bqd/+LieedTGTRmHZYYVCoRgKlCCNIGnb5DdbH+Odpvc5q/I0rpj92QNW2RwoyXiUv7/4ewpLJzL31AuGyFOFQqEYeZQgjRDRdIz/2fQrdnbW8pnpn+LCKecNSbfaB395kmQ8wsc/e8OYme7HtiWJeJp4LI1t29iWxLb7bJbdJy6dPP3SziYAl0fH4zFweww8Xid0e3TVJTnCSCmREqQtndVwbYllSSzLxrZsJ27aWJadsUk6W2O0t8X62SzLzuazLemUCyBxVg2WPedzjFL2ONC7v19+20bTNQyXgWFo6IaWDZ24juHSDthnGHq/9HB9Zr3X0pvuvR5nZ8/19KDpAk0Tx+VvXAnSCNAab+NnG35BW7ydL827kiUVi4ak3Mbaauqq32be6csIlU0ekjIPh5SSZMIkGk4SjaSIhpPEIiknHkkSDaeIRZLEoin6/u0NB26PnhEnIyNYeq9geXvtLreOZdmYaRszbWGaNqZpZdI2lmmhaRrRaDKTtkmnrWy8J6/QBLou0HQNXdfQdeFUepqGZoiMTevNY2TyZOyaLpxNOBWN0ARCkI1rfeJCCHreVxZYCCsNZgphJWnwGYSjJpbUsKSOZQssqWHbAtMWWLbENiVmT8XfIxKmjdlHCGw7IwZSIm2yInMw4Rnu71IgQfTEe5DZuED2sR0Yt6WGhY5k8MIyVHX/UH1WAtB0MDTQddA0ia6BrkncLg2kha7J3k30DW2QYEvHH5nxK5vuu9E3LZx8ABLmnzKJGactGJoLyhElSMNMbVc9/7Xx/5BS8rVF1zOzcNqQlJtKxnnnhUfIL65k3ukXDkmZlmXTsi9Cd0c8KzDRSH/Rsa0D/+K8PgN/0EMg6Ka4NIA/z01xSZB4LJW929M0kanUtWy6d5/WL5+zT3NWrk1ZJBMmqaSZCS2SSSed6rEnTaLhFO2tMceeNA9bMWiayNw1O3fPHo+BwEIXNh7Nxu8x0T0mhkyhZzbp5MCWOjY6FppTEUoNO6VhSUHS1rAl2LZwhMIG2wbLllgWGREYkq/qwGvCQhdWJrSdTZO9oQbebOUGQliO2GEhNAshLQQmSBNhWwjDRstW+k4ohMzaBBJNWM4nIezseXv9sPcLnXx63xD7QCHQDTA8CMODMNzgcuIY7t7Q5cnkcYPhASuNTISxYt2YsSjpeBwzHsdMJhzRljoWRibUMaWBbfixDD+27sfSfWguD2Y6DdJG2jbIzNYn7titPvukE/ajR0idL9q5vky83/4D7T3YaFlf7R7/zT7pmBNPS71fXqvntyldmXKd70z0+f76x+mX1vbLk+5qPerf5UBRgjSMfNC8iV9WP0KBO5+bFl5LeaBsyMre8NenSUS7Ofsz16EbrkGVYduSln1h9tZ3smdXJ427uzDTvX9gbo+eFZrKSfnZeCDPCf1BD/6gG+MgXR6jtUCftG3sRJhUuJtkOEwqGkFPRdHNCFo6gp7qRqTCyEQEmQgjE2FIpw5emBAITxDhdd4Jk1YaLDMTZrZ++TPbEW7UnTvTzJ+9MMDtR3oC4PKDO4B0+cHlA5dvv7iX/IIAsXAETZromOgyhSbTCCvt+GWmDgjZf59t9lbouqtPhe8GI5AJ3Qjd3cfuzgqA0F1guI54oUVFftrbY4fNIwxXf7EZwsl9pW0jkxFkPIxMdO8XhpHxbmRiLzIeRpNpLF1H6IbzmWiGI466kYm7QNOda8/m0R17No/hNGuE5uwTWjaNpiFE5m6gzz7Rk0fTHFs2b5+0EP32lZTk09Ye692XOfZYeNdQCdIwIKXkpYZXeXLns0zNn8wNJ12T05xzubJv13ZqNv2NOUs+SXHFlJyPs21JW3OEPbs62VvvCFAq6SxVHirxM2dBBROqCikuCxAIenC5R/eZlJQSzGRvBZLoLyTZdGa/nQhDMkbP3aYO+PoW6PIivXngDSJ8+WihiQhvkLySEqKWG+ENIrz5aN4gwpsHHv9h/8illGBbvRV+ZnMq/3RWwHpsQnfER3gCCLcf4fE7FfwA+ovG00q87tI8dEbPV6FpCF8++PKBw7/jN54+V90XRLiHuR91lBgRQdq9ezc333xzNh0Oh4lEIrz99tucf/75uN1uPB5nEbjbbruNc845ZyTcGhYs2+IPO9by1z1/Y1HpAq6adwVufXAtmIORTiV454VHyAuVMf/Miw6bV0pJW3OUPfWd7N3Vyd6GLlJJE4DCIh8z55YxcUohE6oK8QfcQ+bjQJBSImOd2F37sLuanLBzH7JrH3ak1anQD4bQHQHx5SG8eWjFk9G9TlxkBCUb9+U7ImAc/BoLS/NID6IyEkL03kX3lz6FQjEIRkSQJk2axNNPP51N33333ViWlU3/9Kc/ZdasWSPhyrCSMJP835bfsrltK0urzmXFjIvQhrgZvfHVtcS6O/jkFbdguPpXsFJK2ltj7N3VyZ76ThobOknEnQo9v9DLjDklTKgqZGJVIYG8kV0FViYi/UUnG28CM9mbUTfQ8ivQQhPRp5yM5ss/qMjg9h+Xo5AUimOZEe+yS6VSrF27lgcffHCkTz2sdCa7+K8N/8fuSCOXz/osH5905pCfY0/NNnZueJVZp3yCkgnT++1rqG3nlT9+SKTbqdzz8j1MmVnMxKpCJk4pJJh/dC/f5oKUEhluxWqrp2N7K/G99VnxIRntzSg0RF4pWkE5rso5aAXlaAUVaIUViEDomOgLVygUA2fEBemll16ivLyc+fPnZ2233XYbUkoWL17MrbfeSn5+/ki7dVT0TJAaNWPceNI1nFgyd8jPYaZTvPT4/xEsKGHB2cuzdsuyeefVOt5/s4FQiZ/zLp7NhKpC8guHV4CkbWJ37MVuq8dq3eWEbfWQigOQAESgyBGd6ac6gpPZRF6J8zxFoVAo+iCkHO63DPrzla98hXPOOYerrroKgMbGRiorK0mlUtx9991Eo1F+/OMfj6RLR8WHrTXc/df/xKt7uOPjNzMtNDzvA7267hE2vP48K7/yTSZNnwNAZ3uMxx9+jz27Opk/1eS00EZnKHOwED1QiB4MoQdD/dKae+BCZSdjJJvqSDXVkWqqJbmvllRrQ/b5jjDcuMum4K6Yhqd8Gu7yabhLJw/qXAqF4vhlRAWpqamJZcuW8fLLLxMKhQ7Yv337dlavXs1LL700oHLb2iLY9uAu42hG19jS5r53fkokHeOfFt901BOkHorWvTW8+Lv/YMEZ5zHvrJXIVIwdb27gr2/FwbY5w/8aUzx1aMWTQWjIWBcy3n2QdyQAlxfhK0DzFzgP+/2FCH8Bmq8A4S9A+AqQ8a5+rR7Z3Zw93BlAUIVeMgWtuAqtpAotvwKx38qz42nUkvJ1eFC+Dg/jwVdNExQXD3xk8Yj2mzz55JOce+65WTGKxWJYlkVeXh5SSp577jnmzh367q7h4oOWzTRE9nLV3MuHTYzMdIq31/8WfyCPeSEv3U/fy9s1IXYkZ1Ps6uQT89oInbAUfdICNH9B9rjsOxixLmS8yxnJFuvqk+7Cbt+NvWdLtpttf0ReqSM8sz6GXlKFVjzFETA1mEChUAwDIy5I//Iv/5JNt7W18bWvfQ3LsrBtmxkzZrBmzZqRdGnQWLbFuprnqQiUc+oQTQXUF5mKY+6pZuMbfyTc0cyp3igNr/yR1+IX0pkKctKJAU5fdhaG6+BDyvu/g3H4bkRppnpFKt6F8ATRiycj3P4hvy6FQqE4FCMqSOvXr++Xnjx5Mk899dRIujBkvN30Pk2xZr5y4heHZGi3lBK7Yy9Ww0bMho1Y+z6kMy3ZGQ9QFSogVvk53tisYbh1Pr1yDlXTi4bgKhyE4UbklUJeKWNjelaFQnE8ooY6DYK0bfJc7QtU5U1kYemJOR0jbdtphUQ7sKMdyGgHMtYbt7uakNF2ALTQJPT5F7C5ehtezaK74FI++KCbaScUcc6ymQSCI/sOkUKhUIwESpAGwd/2vk17ooNVsz+HEAKZTjqiEuvoIzjtyGinE491IGNdBw4yEDoiUIgIhNArTkCfMBdj8gK0YDGbXn+W7q52LM8naNzRzennTuPC5fNobYuMzkUrFArFMKMEaYCkrBR/qnuRmYGJTKutJvrKw9ht9QdmdPvQAiGEP4QWmuDEAyE0fwgRdOzCl3fQl0Dbmxqofut5UvYUEBNY+YW5VEwqQGhqMIFCoTh2UYKUI1La2M01/PnDZ+lOh1lVU0868QF6+UzcSz6LFixxBCcjPMI1uHdwIt1xXvz9L7Clh+Ipn+CTl5yExzt0c+EpFArFWEUJ0mGQtom1dztm3buYde8RT3Tx0pRi5uBl9pJVGFMXofmHbrj37roOXnn6D7isNqYsuJQzLzhFDbFWKBTHDUqQ9kOaSczdmzFr38Ws3+DMwWa4MSafxN9CPuLd21l56mrceYefzn4gWJbNX1/8iB3vbyXftYXyKSdx1oWfGLLyFQqFYjygBAmQyShm/QZHhBo2gZUCTwCj6mSMaYsxJs0nYqf5yxv/xqKyk5g8RGLU3p3g9U2NbPj7HorjaXzG30lLg6frp7D+v98gz+8m6HMR9LvI87uoKMlD2DZ5/ozN5yLP78br1gfUkpJSYnV3k25uJtXcRLqlKRNvJt3cjDTTCF1HGAZkQqEbmVBHGHo2jW5k0v3zRQqDJDU3eiCAHgiiBQLowaCTDgYRHq9q/SkUin4c14JkNX9E45+fIV67GaSF8Bfimv0xjGlL0CtnOatAZnh+x/OkrDTLpx3dcuGptMV7O1p4fWMj1XUdSGCxYRD01+I2O3HN/DRLvFMIx1OEY2nauhPU7esmEk9jHmT5cABdExmBclOU76E430txvpsSLUUoHSYQ78LoasNsaSbd3ESquQWZTPQWIASu4hJcZWV4Tz0NzetBmhbSNJGWhbRMME3HZmVspomdSiGtuLOKaja/iTRNwqkUdiJxUH8dp3V0vyNOWbHqmw4E0Pz+XiHMiqHRRyx79mUEsmd/j5AqwVMoxhXHtSCZe7ciu9twn7QMY9pitNJpBx311pHo5K973uD0ysVUDGIZciklNY3dvL5pH29VNxFPmhTne7nk7KlU+dy88+cNeIxNTJy5kLMuufCgFamUkmC+j9qGDiKxNJGMYIVjaSLxNMnWVoo+fBf39mb80Q7yU2Fc0kICEcBCI+LJIx4MYU+aj1ZSire8grxJEyiqqqQ4FMDQh27Zh9LSPJr2tmPHoliRKFY0gh2NYEWjzhaJYEcduxWNYra1kty1CysaQaYOsaT4QOkRKJcLzeVGuFxO3N0bF2437Xl+UpZAuDP53C6E0ZMvk1fvs7S0lolrGmjCCUUmnVlSWvTL4xwn3G40n8/ZDjHDhkJxPHNcC5Ln5OWUXnDlEScq/GPdi0gpuXjq0gGV3xVJ8rct+3h90z72tkZxGxqLZ5fysQWVzJ4SQgB/+OV75AcbkSmbRed97pB39UII/F4XZYU+ygp7VydNNTXR/sc/0/3G6yAl7soJuKqmoRWXkswLEfEW0G7k0WR7aAunaOtO0NaVoKsthWyzoXo3sBsBFOY5rauA18BlaBiGhkvXcBnOZvSJu/QD9/fEDUOjI27S3BLBtCRp00PadpF25WPmSdI+m3ShTdq0SFsS07RJmzZpyyJtSqxUChGPYZhJgm6NfK9G0O1sAbcg4NbwG6BjQ6a1lm29mb0tOCwLmU5jm2lkKu3E0ylkKuXEYzHszk6sfSZmItVvH8M857DUdYTXh+b1oft9GH4/ms+H7vOj+bxoPj+a14fmd/I4+3xEwiGSXQnQdISuITTdETxdc2ya5gixpiF0vVcUcW5qTMsmbUpchsDQteOuFWn6F4sFAAAgAElEQVRLiWU5n4NlSyzLxrQktq7TFU6iawJNE+iawNAFuqahDeJ1C2d5ezvTw2A5v8We3oVs2nZ6HzI2aVlg2yBE73fXJ+y5uUnYMdKdcee7z/wG+n3/fb7z8cZxLUi50Bxr5Y3Gdzhn4hkU+448XY9p2WzY2cprGxvZVNOOLSUzJuZz9admc+qccvze3o+8obad1qYIE4r2ESydhj8v9xF7yT27aX92HeF33kIYBoXnfoLQsotxFRfndHzatOkIO+LUmhGptq4Ebd0JOiLJjJBYjlCYzh9tyrSGvJ4WgMt1MIEzkNKguy1Fdyx10PN63ToFQQ8FAR8FAbezFbgpCHgoCDrpoM+FbdokUibxpEUiZZLIhPGUE6JpdHTGSaQs4imTRNIkmUhhJlOYiQRWMgVSoiERSDTphKLHJiUadsYGAjubR+uTz7BNPHY6s6Vw22k8yRSeeBpvazteaeLN2F1mCsGBF90wiM9YArZwPLQRyExcCuFckRAgHHs21AQIp+XXExea6G0dCg0pNKeMzDF9y5MINEPHtGS/ffZ+eey+Th4Y7fe9y77/yz4Zeip/KZG27VTqtg09aelswpaZ79HOfE8SIe1Df699bNn0/rY+vwEnv53dhou6YSu5P61LPslZN35xhM7moATpCDxX+wK60Fk25ZOHzVffFOa1jY28Wd1EJJ6mIOhm2emT+diCSiqLAwc95v03GwgEUiQizcxZfFZO/iRqa2h7di3RD95HeLyEll1E6IILMQoGNvzcZWiUhfyUhQY2gapl25imJG1lWjX7t3Iy6fx8L7Fo8rAtKkPX0DVxxLt025aE42m6Ikm6oim6Iim6ok68O5qiM5KivjlCdzRJPGkN6HqEAL/Xhdel4XUbeD06Pq+LonxvNu11G7iNw7cO+7UeD5JH1wSmJYkm0kTjTjdrNGE68USa9riZsWX2x9IkY3HMaAyRTuKxU3jsNC7bzFSAdrZyNITEowtcOrg1cGU2twaGJnEJMIQTNwRo0kbaNrblhDIT2raVrdB7bEgbaTsVPpaNTEuw05kK2O5fKWfiep/K3N2vspZZERCQDY+WAwW1R0x7xDXTlapr/YVVaM5zYk2gGwamLTNC6Qho5hajV0QRWAjSwrlV6BH4vpvUnJaKzLRcyYRS15243tOaMZz9mbgw9N4WDhJh94iojZDOdyIyaY9bJ5VI9bMdkPcgNzMDZfLZpx91GQNFCdJh2BvZx9+bPmBp1bkUePIOmmfjR2088ZePqG+OYOiCk09wuuTmTwuhH6bZ3NzYzZ5dncyeFaalDibOXHjIvFJK4h9uZ8sDf6Tzgw1o/gDFn1lJ4flL0YMDX3PkaNA1Dd0NniNMwzqUa7Zomsi2gI5EMm3RHU1lhCtJJJ7G7dLxunV8PYLjNvC6dbweR2jKyvJHZH0ZlyEoDHooHOBchKm0lRWvvHwfsWgCr9vA49LxuLXD/s5Gk/Gwbk8PytexgRKkw7CuZj0e3cMFUz5xwD7Tsnny1Rr++GY9lcV+vnDBLE6fV07Ql9vD6vffbMDtMbCTuygsnUiw4MCuNiklsc2baHt2LYmdO3AVFlLyuc9TeN55aF7fQUpVeFw6pYU+SguPnc/H7dJxu3RCeZ5jujJSKJQgHYK67no2tG5h+bRlBFz9u7XauxP81zNb2Lm7i0+cPIErl56Ay8h94YaOthg121tZeGoxDRvqmH/Gsn77pW0Tef9d2p9dR7J+F0ZREaWr/pEZKy+mvXuIRqApFArFGEMJ0iFY+9F6gq4A500+u599U00bP19bTdqyuf4z8zhjXsWAy/7grQZ0QyNU0E4DkoknON110rIIv/0m7c89S6pxL67ycsqvuZb8M85CGAa6xwMoQVIoFMcmSpAOwocdO9nWsYPPzVyO13AmSbVsm6dereXZN3YxqTTA6pUnHnKwwuGIhJN8uLmJuSdX0lL/IoGCYgqKKuj8yyt0/PFZ0q0tuCdOouL6G8lbctq4Hb6pUCgUA0UJ0n5IKXnmo/UUego4Z+KZAHSEk/z3M1v4sKGTjy+sZNXSWbhdg1tbdeM7u5FSMv/kEl56ZDszT/44HX96jrYnH8c7bTqlV6wicNJCJUQKheK4QwnSfmxu20pt9y6unH0pLt3Fltp2/mftFpJpi68sn8eZJw68i66HZCJN9QeNzJxbRrS9BtuyKC+YQPsjPye45FQqb7jpuHtRUaFQKHpQgtQHW9qsrVlPia+Y08uX8ORfa1j3tzomlDhddBNKBt5F15fN7+0lnbJYdMZktr/zBzy+IOln/4Tm9VG26otKjBQKxXGNEqQ+vN+8kT2RRj4//TJ+8vuNbKvv5OwFFfzjBbPxuAfXRdeDmbbY+Pc9VE0vorDIQ2PtFsr9pSRrq6m8fjVGfv4QXYVCoVCMT5QgZbBsi3U1z1PkKuWJtUmSyRjXXjyXj51UOSTlb9u0j0QszaIzJtNU/yFmKom3ZifBRYsJnnrakJxDoVAoxjM5PTn/3e9+R3d393D7Mqq82fh3muOt7NsyiaDXxXevXjJkYmTbkg/e2k35xHwqJxewe+cGdCnIt1yU/aPqqlMoFArIUZBeeOEFzj33XL7+9a/z5z//GdM0h9uvEaUtHOPR6j9iRwo4deJJfPfqJUwsHbopeT7a1kK4K8Gi0ycjpWT31vfI605RccUXBjwHnUKhUByr5NRl9+CDD9Lc3My6det44IEH+M53vsPFF1/MypUrOemkk454/O7du7n55puz6XA4TCQS4e2336a2tpY77riDzs5OCgsLuffee5k6deqgL2igbNrZyj3r/oBVGeP8CZdy2eJ5Q9pikVLy/hv1hIr9TD2hmMbNfydtpSgPTSDvjDOH7DwKhUIx3sn5GVJZWRnXXnst1157LVu2bOFb3/oWjzzyCFVVVVx++eWsWrUKr9d70GMnTZrE008/nU3ffffdWJYzK/OaNWtYtWoVK1as4Omnn+bOO+/koYceOsrLyo03t+zjf5/biHfhDqYEpvIPS84Y8nPU17TT1hLlvE/PBinZuf4JhJDMuvI61VWnUCgUfRjQ25fvvvsua9as4ctf/jJ5eXncdddd3Hnnnbz99ttcf/31OZWRSqVYu3Ytn/vc52hra6O6uprly5cDsHz5cqqrq2lvbx/4lQyCtGUz77RupJHksjmfHpZzvP9mA8F8DyfMK6Pj5RdpM8OUFFbiLx+a51MKhUJxrJBTC+knP/kJ69atQ9M0VqxYwWOPPcbkyZOz+0899VROPz23tTNeeuklysvLmT9/Pps3b6a8vBxdd4ZU67pOWVkZjY2NFBUdeTG8HoqLB/e858JzJrF23SYWly/g9JknDqqMw9FQ10FjQxcXrphHoUiw5dknSE1yceL5n6K09ODLWRyJwR43Gihfhwfl6/CgfB19chKk1tZW7r33XpYsWXLQ/W63m0ceeSSnEz7++ON87nOfy93DHGhri2DbA1+Q6tnaF4im41w48ZPDMqX/y3/ahsdrMHl6iOr/99/pDDgLheWXnzCo842npQeUr8OD8nV4UL4OLZomBtVQyKnL7tZbb2XOnDn9bJFIpF/X2v77D0ZTUxPvvPMOl1xyCQCVlZU0NTVlnydZlkVzczOVlSPTnVXqK+aKBZ9hUt6EIS+7vTVK3Y42FiyeSOzNV4lv20p4QojSCdPx+o/NuxuFQqE4GnISpBtvvJHdu3f3szU0NLB69eoBnezJJ5/k3HPPJRQKAVBcXMzcuXNZt24dAOvWrWPu3LkD6q47Gk6rOIVL5100LGV/8FYDhktjznQvLb9/FDF3NpF4NxNnHnlUokKhUByP5CRINTU1B7SA5s6dy86dOwd0sieffPKA7rrvfe97PPzwwyxbtoyHH36Y73//+wMqcywS6U6wY0szc06qoOvRXwOS9JL5AExSgqRQKBQHJadnSKFQiIaGhn4DGRoaGigoKBjQydavX3+AbcaMGTz22GMDKmess+FtpzU5g71EqrdQ9oUv8m7jJgpLJxI4yFLlCoVCocixhbRy5UpuueUW/va3v9HQ0MDrr7/ON77xDT772c8Ot3/jjkQ8TfWGRmbMKCD2zO/wzZ6DZ8kSWvfWqtaRQqFQHIacWkirV69GCMF3vvMdmpqaKC8v57LLLsv53aPjic3v7sFM20ysfx1pWZRffS0NtZsBycSZC0fbPYVCoRiz5CRIuq5z880395v+R3Eg6ZTFpnf3MLFIoL/9NiVXrMJdVsbu1x4nWFBCQYl6GVahUCgORc5TB1mWRUNDAx0dHUjZ+87PKaecMiyOjUe2bmwkETeZUP9nvDNPoPD8paSScZrrP+SEReeqqYIUCoXiMOQkSBs2bOCWW26hs7OTVCqFx+MhlUoRCoV47bXXhtvHcYFl2Wx4ezdFWoSC2D4qrvkBQtNorK3Gti013FuhUCiOQE6DGu6++26uvPJK3n//fQKBAO+99x7XXXedeobUh51bW4h0J5m8+w2KV16Ku6ICgD07N+D151EyYeroOqhQKBRjnJwE6aOPPuK663pnpxZCcPPNN/Pggw8Oq3PjBSkl779eRzDdycQKD6ELlgFgmWkaa7cyYcYChBjQPLYKhUJx3JFTLRkMBonH44Azu0JNTQ2RSIRIJDKszo0Xdu1so6MjwZTOzVRc82WE5nysTfXbMdNJNdxboVAociAnQTrvvPN48cUXAfjsZz/L1VdfzWWXXcYnP/nJYXVuvPD3P2/Fmw4z7/yFeCb0zou3e8dGXG4vZVWzRtE7hUKhGB/kNKjhe9/7XjZ+ww03sGDBAqLRKOedd95w+TVu2L19Ly1dNvO0fRR/6ktZu23b7K3ZROW0+eh6zoMZFQqF4rjliC0ky7K4+OKLSaVSWdtZZ53FBRdcgGGoinbbXzah2SZLvnABIrOuE0Dr3hqS8SiTTlDddQqFQpELRxQkXddJp9P9BEnRS2dHkiAxAlOn9LPv2bkRTTeomDpvlDxTKBSK8UVOz5CuvfZabrvtNjZs2EBTU1O/7XhGSkk4bZDvFwfYd+/cSMWU2bjcnlHyTqFQKMYXOfW59SwJ8corr/SzCyHYunXrkDs1Xog3NpMwAoRK7X72zpbdxLrbmX/GslHyTKFQKMYfOQnSxo0bh9uPcUnztloASqrK+tl379iIEIIJ008cDbcUCoViXJKTILnd7uH2Y1zSWtcEFFJ6wqR+9j07N1KilipXKBSKAZGTIH3pS1865MSgv/jFL4bUofFER0sUIQsoLO0VnnBHC11tjZz8CbVWlEKhUAyEnARp2bL+z0JaW1t54oknWLFixbA4NR6Qtk13zCaQZ6LrvWND9ux0ujcnzVDDvRUKhWIg5CRIV1xxxQG2iy66qN8Ls8cb6eZmInoeRfmufvbdOzdQWDZJLVWuUCgUA2TQM35OnjyZ6urqofRlXBGrrSHuyqeosjBri0e6aGusU3PXKRQKxSDIqYW0du3afulEIsGf/vQnTjzx+B1F1rpjN1KUUzKld4TdnprNAGrtI4VCoRgEOQnSww8/3C/t8/mYO3cu11133bA4NR5o39sBlFNUFsza9uzYQLCwlIJitVS5QqFQDJScBOnRRx8dbj/GFdK26ehMQiEUFvkBnKXKG3Yw65RPqKXKFQqFYhDk9Azp2WefZfv27f1s27dv57nnnhsWp8Y6qcZGonoQvwdcbmdC1caaLWqpcoVCoTgKcmoh/fu//zuPP/54P1tpaSmrV6/m4osvzulEyWSSe+65hzfeeAOPx8PJJ5/MD37wA84//3zcbjcejzPn22233cY555wzwMsYWRJ1tcRchYSK/Vnb7p0b8QbyKa6ccpgjFQqFQnEochKk7u5uCgoK+tkKCgro6urK+UQ/+tGP8Hg8rF+/HiEEra2t2X0//elPmTVr/CxiF6+tJeqewNTMCDsznWJfXTVT5p6qlipXKBSKQZJT7TljxgxeeumlfraXX36ZadOm5XSSaDTKU089xS233JJ9vlJSUjJAV8cOnfV7sTWDotIA0LNUeUp11ykUCsVRkFML6dZbb+XGG2/k/PPPp6qqivr6el555RV+9rOf5XSShoYGCgsLeeCBB3jrrbcIBALccsstLFmyBHC66aSULF68mFtvvZX8/PwBXURxcfDImQ5DaWnuc87ZpklHSwzKYeqMEkpL89j4l624vT7mLzoFfZgXLRyIr6ON8nV4UL4OD8rX0Sen2vP000/nqaee4plnnqGxsZGpU6fy5JNPUlVVldNJLMuioaGBefPm8c1vfpMNGzZw44038sILL/Cb3/yGyspKUqkUd999N3fddRc//vGPB3QRbW0RbFsO6JgeSkvzaGkJ55w/Ub+LqO4IoNAFTU2dfLTlfSqmzqO9Iz4oH3JloL6OJsrX4UH5OjwoX4cWTRODaijkJEi2bVNVVcXXvva1rE1KiW3baNqRe/0qKysxDIPly5cDsHDhQkKhELW1tSxYsABwZhRftWoVq1evHvBFjCTJujqirgK8Xh2f30Vzww5SiaianUGhUCiOkpyeIV199dW89957/WzvvfceX/rSl3I6SVFREaeffjqvv/46ALW1tbS1tVFWVkY47Ci9lJLnnnuOuXPnDsT/ESdRV0vMW0So1FH/fbu2ITRNLVWuUCgUR0lOLaRt27Zxyimn9LMtWrRoQKvFfv/73+fb3/429957L4ZhcN9995FKpbjhhhuwLAvbtpkxYwZr1qwZ2BWMMPG6WqKejzMhM+S7s2Uv+aFytVS5QqFQHCU5CVIgEKCjo4OioqKsraOjI/vuUC5MnjyZX//61wfYn3rqqZzLGG3sdIpwYyvpKUb2HaSutkZKKnMbbahQKBSKQ5NTl93SpUu5/fbb2bVrF7ZtU1dXxx133MGFF1443P6NKVK7dxMznNEtoRI/6WScWHc7BSVq7jqFQqE4WnJqIf3TP/0TP/jBD/j0pz+NZVnous7KlSu57bbbhtu/MUUiM6ABIFTsp6utEUAJkkIxAliWSUdHC6aZGvKym5s1bNse8nKHg7Hmq2G4CYVK0fWjf+UlpxJ8Ph/33HMPd911Fy0tLZSWlmIM8/s2Y5FEXS3xQCkut04gz0NT3V4ACksmjLJnCsWxT0dHC16vn0CgYsgnMDYMDdMcO5X84RhLvkopiUa76ehooWQIbswHpCq2baPrOm1tbVlbeXn5UTsxXkjU1RLLO53CIj9CCLpaGzFcHvz5odF2TaE45jHN1LCIkWLwCCEIBPKJRDqHpLycBKm2tpY77riDjRs3HrBvICPtxjN2Mklq7x6iswNM7hnQ0LqXgpJKNX+dQjFCKDEaewzld5JTTfq9732PefPm8de//pVgMMirr77KpZdeyr/+678OmSNjnWRDPSYGcVMnVOJHSklna6NajE+hUCiGiJwEaevWrXzrW9+itLQUKSUlJSV8+9vf5oEHHhhu/8YMibpaou7eAQ2JWJhUIkqBen6kUByXXHbZJXzxi5/vN8DgsssuoaZm52GP+/3vf0tHR/twuzcuyUmQXC5X9kMvLCxk3759CCH6PUs61knU1RIvcMQnVOKnq1WNsFMojnfi8Tjr1w9sodLf//4RJUiHIKdnSIsXL2b9+vWsWLGCpUuXcsMNN+DxeDjttNOG278xQ7KujmTxIjRbkF/oo6nWGWGnBEmhGHle39TIaxsbh6w8IUBm5mf+2EmVnL0gt7/ra6+9nl/84ucsXboMl8uVtbe2tnL//ffR1LSPZDLJ0qXLuOqqa/nVrx6ktbWF73znm7jdHtas+SHTpk0fsusY7+QkSD/5yU+y8dtvv50nnniCaDTKZZddNmyOjSWseJxU0z5ilUUUeHxomjPCzuPPw+s/NqeBVygUR2bOnLnMnj2HJ5/8A5///JVZ+w9/eCfXXHMdJ598Cul0mltuWc3cufO4+uovs3btU/zwh/cyffrMUfR8bJKTIOm63i/+D//wD8Pm0FgkWb8LpCRseSjtM8KuULWOFIpR4ewFubdicuFo3u25/vrVfO1rN7J8+QoALMvm/fffpbOzdyh0LBalrq6OU089Y0j8PVY5/t5uHQSJulosoROJ28wq9iOlTVfbPqYvOHO0XVMoFKNMVdVUzjzzbB599DcAaJqGEIL//d+HjssJBI4G9QJNDiTrakmVTEZKCJUEiHa1YZkpNUODQqEAnGdJTzzxGLFYDCFg4cJFPPzwL7P7m5r20dbWCjiTVUcikVHydGyjBCkHEnV1pCqcB4+hYj+daoSdQqHoQ1lZOcuWXUx3dxcAd975A2pra7jqqsu56qrLWbPm21kRuuyyK7jnnru45ppV1NbWjKbbY46c2pP33nsv3/zmNw+w//jHPz7mJ1i1olHSLc3EZ1VAHAqLfGyvdQQpX70Uq1Act/zhD2v7pb/61W/w1a9+I5v+/vfvOehxl1yykksuWTmsvo1XcmohPfroowe1P/bYY0PqzFgksasOgKieR16BF8Ol09W6l0BBsVqUT6FQKIaQw7aQ1q517gAsy2LdunXInoH6QENDAwUFBcPr3RggWVcLQHdSJ1TiA5xF+VR3nUKhUAwthxWkhx9+GIB0Ot1vtVchBMXFxdx9993D690YIFFXi1FaRldngskzirHMNOH2ZibNPGm0XVMoFIpjisMKUk9X3aGeIR0PJOrqsKfOweqWhIr9hDuakdJWk6oqFArFEJPTM6SbbrqJRCIBOAsyrVu3jueeG9j8TeMRs7sbs72NeMkUwJnDrrO1Z8ogNeRboVAohpKcBOm6665j505nBtv777+f//zP/+SBBx7gvvvuG1bnRptkZkBD3FcMQKg4QHdrI5qmkxcqG0XPFAqF4tgjJ0Gqqalh3rx5ADz99NM8+OCDPPzwwzzzzDPD6txok6irBSEI2178ATcer0FnayN5ReVofaZTUigUCsXRk9N7SJqmYZomu3btwufzMWnSpMxa6tHh9m9USdTV4i6voLMrSWGfOexKJqjZeRWK453LLrsEn8/Hr371OzRNy9ruu+8nh5049fe//y0XXPApQqGiI5bvdrtxudwkEnGmTZvOF75wNYsWLcrm2bDhff7rvx6go6MDy7JYtOgUvvrV/4f8/PyhucgRJqcW0tlnn83tt9/OXXfdxUUXXQQ4raaysty7rZLJJGvWrOHCCy/kkksu4bvf/S7gLI9++eWXs2zZMi6//HLq6uoGfhXDRKKuDveUqXS2xQiV+Ekn48TCHRSUqgENCoVi+NdD+uEP7+VXv3qERx99iosuWs7tt9/C5s2bANi9u4F/+Zfbuf76m/jd757g0UefJBgM8t3v3jHg6xgr5NRCuvvuu3nssccwDCO75ERzczOrV6/O+UQ/+tGP8Hg8rF+/HiEEra3OvE5r1qxh1apVrFixgqeffpo777yThx56aBCXMrSYnR1YXZ0wcTqpjRahYj9dbZkpg9QIO4ViVEl/+Drp7X8dsvKEENn3LF2zP45r1tk5HTeS6yGde+75VFdv4be//TV33fVvPPTQL/j0p1ewaNFiwOnJuummW/j851ewYcP7LFy46Agljj1yaiH5fD6uuuoqVq1aRSwWA+DMM89k5crcpr+IRqM89dRT3HLLLQghACgpKaGtrY3q6mqWL18OwPLly6murqa9ffRXU0xkWmrxAkd8QsW9q8SqSVUVCgX0Xw+pLz/84Z1cdtkV/PznD/Hggw/z5pt/45133uTqq79MSUkpP/zhvfzyl78d8OJ88+adSE3NRwB89NFO5s9f0G+/YRjMmjWbnTs/PLoLGyVyaiFFIhHuueee7FDvDz74gJdffpnq6mpuvvnmIx7f0NBAYWEhDzzwAG+99RaBQIBbbrkFr9dLeXl5dr0lXdcpKyujsbGRoqLD968ON4m6GhCCqJEHtBEq9rOtphHD5cGfHxpV3xSK4x3XrLNzbsXkwvhZD6l3tpy+M+ccK+QkSN///vcxDIO1a9dy6aWXAnDSSSfxb//2bzkJkmVZNDQ0MG/ePL75zW+yYcMGbrzxRv7jP/7j6LzPUFwcPKrjS0sPXPW1ee9u/FWT6UpKPF6DKdOKef/FZoorJlFWNnpTJh3M17GK8nV4OF59bW7WMIzhW6BgMGXrusb06dM566yP8dhjvwXA5dIRQvDLX/4aw3Ad8rhczrd/vu3btzJjxkwMQ2PWrFls3bqZ888/P7vfNNN8+OF2vvCFLw7rZ7U/mqYNyXedkyC99tpr/OUvf8Htdme73IqLi7PPgY5EZWUlhmFku+YWLlxIKBTC6/XS1NSEZVnouo5lWTQ3N1NZObBnNG1tEWx7cHcLpaV5tLSE+9mklIQ/3Elg4cns3d1FYZGPlpYwLXsbmDTzpAPyjxQH83WsonwdHo5nX23bHnQr5kgMtoVkWY5PX/rSV/jyl7+IZVnYtmThwkX88pf/xzXXXAc46yEZhkFxcQmBQICuru6cztdTPsCrr77CE088xv33/3+Yps2qVVezevW1nHbamSxatBjbtnnggfuZNGkyCxYsGrbP6mDYtt3vu9Y0MaiGQk6CFAgE6O7upqSkJGvb9/+3d+9xUdZp48c/M8CAIMhJEQUFTQ5qHik1T4G2abqpeczCMtNdf0m5LpapWbq6hvqoj4i51u8p23yyekwNpdUn7bDbVlpo5VnJA3JSDgLjMBzmvp8/RidJRMU54Vzvf3xxz8091z3z9XXx/c53ris/v9bP9QkMDKRXr158/fXX9OvXj9OnT1NUVERERASxsbHs2LGDESNGsGPHDmJjYx2+XFdTXIRJX45XRCQlhy7Ttl0QRkM5VcbLUqFBCHGdq/2QNm821/9csOAvrFmzkkmTxgPg7e3Dyy8vICgo2NIPycvL66abGubPf8my7TsiIpLly/+Tzp3vpaZGoU2btixevIy//S2NS5cuYTLV0K1bDxYvTrHLPduCRr2Fhci0tDS++eYb/vznPzNt2jQ2btzIihUreOCBB9i5FuEAAB5kSURBVHj22Wdv6Ymys7OZO3culy5dwt3dnZkzZzJw4ECysrKYM2cOZWVl+Pn5kZKSQrt2t/dBn7VnSOU/7CfvjTRCZs/nv7fm0Du+HaEtL/PllnU8OGYGIW2iGvRcd8qV/zq2JYnVNqwda37+WVq2bGu1613rTj5DsjdnjPW3741NZ0jTp0/Hw8OD5ORkDAYDM2bMYMKECUyZMuWWnyg8PLxWxfCr2rdv73R9lYynT4ObG5c9A4CcKzvszKWTpO2EEELYRr0JaceOHQwfPhytVsu0adOYNm2aveJyqMqzZ/BsHUZpaRVg3vJ9PCsXT29fvLwbzwfKQgjn9vbbb/Lll59fd3zVqrU3reRwN6o3IS1YsMCyEcFVqKqK8cxpfO/rRU6RATc3Db7NvCgtzMNfZkdCCCuaPHkqkydPdXQYTqPefYF34z73m6m+cAGlosK8oaHIgH+gNxqNSmlRvmxoEEIIG6p3hqQoCt9++229ialPnz5WD8qRjGfNLcs9IyIoOZxPSCtf9JeKMNVUSckgIYSwoXoTUlVVFfPmzbthQtJoNOzZs8cmgTlK5enTaNzdcWvekvLSM0TfG/JrDTuZIQkhhM3Um5CaNGly1yWcmzGePYNnmzaUllUDEBjsQ2nhMQD8glo6MjQhhLir2a+2RCOgKgrGs2fxbGv+/AjA/0pRVZ9mQXjoPB0coRBC3L3qnSG52qaGqvx81EojXhGRnC28jEYD/gFNKC3Mle8fCSFqsUeDvurqKj7+OMNSgDojI52//nUhf/rTbEaPNleBOHr0MH/7Wxq5uTn4+TXD29ubZ56ZRrduPZgxYxoFBQX4+PhQUWEgNLQV48ZN5IEH+lme55dfTrF27X+Sk5ONoqhER0eTlDSLkBD7rwjVm5AOHDhgrzicQuWVDQ1eEZGU7LuEn38TwER5yUXCOnR1bHBCCIvv8n7gm7z9VrueRgNX//7uE3ofvUJ73tLvXW3QN3TorX895sMP3ycu7v5b+p5RUFBz9u37hj59zAkkIyOd6OgYy+NZWaeYPXsmr7yyiF69zBvMcnLOc/Lkccs5M2cm07dvfwAyM7/ntdfm8ec/v8TAgQmUlZUxc+ZzzJw5m4SEwQB88MEmZs2awcaNm3F3v6XaCVYjS3bXMJ4+jUanQxcaSkmRgYAgb8pKLqCqiuywE0Jc52qDvurq6lrHCwsLmT//RaZOncSkSeN5993/AqjVoO/ppydy+vQv9V7/kUeGk5GxAzAnGqPRSPv2HSyPb9q0keHDR1iSEUDr1mE8+OCgOq/Xo0cczzwzlb///R0Atmz5gO7de1iSEcD48U/g7e3DZ5/tuvUXwkrsm/6cnPHsGbzaRqCiobS4goh7gigtzAVkh50QzqRXaM9bnsXciobWh7u2Qd+4cY9bji9evICnn36Wbt16UF1dzQsvTCc2tiNPPTWF9PRtLF6cUu+y3lXdu8exdev/UFZWxj/+sZMhQx7hxIlfZz8nThy7YfK5kY4dO5OaugqArKyTdOnS7bpzOnW6l1OnTt7Wda1BEtIVqslE5bmzNBsYT2lJBYqiXtnQcAit1g3fgBaODlEI4YRs2aBPo4GEhIfYs2c3n322i/Xr/6tWQmqIa7cGONs+AUlIV1Tl5qJWV+MVGUnRlR12AUHeHD+Vh29gCNorHyoKIcS12rSJoE+fvnzwwSbA3KxOo9Hw1lvvWuUzmCFDhvGHPzxN167dadbMv9ZjUVExHD16mAEDHrzl6x09eph27doDcM89URw+/PN15xw5cohRo8bcUdwNIZ8hXXG1QoPXNVu+zVW+c+XzIyFEvZ55Zhoff/wRBoMBjQa6du3Oe++9Y3m8oCCfoiJzQ1MfHx/0ev0tX7t16zCmTv1/lmZ/13r88UTS07eyf/93lmO5uTl88UXd3x/98ccDvP32Bp544ikARo8eR2bmD+zd+5nlnA8+2IReX85DDw255RitRWZIVxhPn0bbpAkeLVpQsv8EPr46NFRjKC+hWXNJSEKIG7NVg76rRox4rM7jHTpE8frrq3jzzXUsX/5XvLy88PcPYMqUP1rOWb16BW+++QZGYwUhIaG8+OJ8y647P79mrFqVRlraatavT0VVzddcuXKt3XfYwS026HN21mjQd3bxQrReXoQnv8T/vJOJp5cbfQb4smfzavqNmErr9vdaOeqGx9oYSKy24cqxSoM+M2eM1VoN+mTJDlCqq6nMPmfeYaeqXCo2EBDkw6VCcw07f9lhJ4QQNidLdkBVTg6YTHhFRnK5vJLqKhMBweaSQe4ennj7uV6jLCGE7UmDvtokIQHGM+Yvp3m1jSSv8NcNDcdOmksGaTQaR4YnhLhLSYO+2mTJDjCeOYPWxwf34GDLDrtmgU0oLcyTL8QKIYSdSELCXMPOKyISjUbDpSIDnl7uaNQKqoyXZcu3EELYicsnJFNlJZU5OXi1jQCgpNBAQLA3ZcX5AFLlWwgh7MTlP0MynDkLioJnRCQAJUUGIqOCKC3MASQhCSGuN3XqU1RXV1NTU0129jkiI82VD6Kiopk799XbutaXX+4lJKQlMTEdbRFqo+LyCan85CnA3HKiwlCNsaL6SoWGPLy8ffHy9nVwhEIIZ/PmmxsByMvL5dlnE3nnnf9u8LW+/PJzunTpKgkJOyakhIQEdDodnp7mrqvJycn079+f6OhooqKiLA2uli1bRnR0tL3CQn8qCzc/P9wDArh4vhQwd4k9fjJPZkdCiNu2Y8d2tm/fgslkwtfXj+TklwkPb8OPPx5k9eplqCqYTDU8/fRUvL2b8M03X3PwYCbbtn3MxImJ/O53Qx19Cw5j1xnSmjVriIqKuu745s2b8fHxsWcoFvpTp2ptaADwD2xCWVEe7bv0dUhMQoj6lf37a0r/9ZXVrqfRaCyVr5v1G4DfAw37v5+Z+T3//OcXrFv3//Hw8OBf//qKlJTFrF27gffee5vExGdISBiMqqro9Xp8fX3p06cvXbp0ZeRI+xczdTYuvWSnGI1UnM8hsJu5r0pJoQF3Dy0a9JhqqmWHnRDitnz99VecOHGcqVPNxUtVVcVgMP+h2717HO+88ybZ2We5775edOzY2ZGhOiW7JqTk5GRUVaVnz57MmjULPz8/ABITEzGZTAwYMICkpCR0Ot1tXbchNZMASg9ng6oS0rUjgc19uVxeRfMQX6g29zGJ6HAPzZs712dIzhZPfSRW23DVWC9c0OLubl7aDxzQn8AB/a127YZyc9MCGktcGg2MHPkYU6ZMu+7cSZOeIj4+nv3797FyZQp9+/Zn6tQ/otGYW1ZcvcatuJ1z7UGr1VrlvbZbQtq0aROhoaFUVVWxZMkSFi1axIoVK/jiiy8IDQ1Fr9cze/Zs0tLS+NOf/nRb125ocdWSg4cBqAwI4eLFcgryyggNb0b2L+bKDSatn1MVsnTlwpq2JLHahrVjVRTFZkVFG1qw1GRSANXyu3369CclZTHDho0gOLg5JpOJkydPEBMTy7lzZ2jTJoJHHw3Dw0PHnj27qalRaNLEh7Kyslt+fmcsrqooSq33uqHFVe2WkEJDzctfOp2OiRMnMn369FrHmzZtytixY3n77bftFRKV2dnogoJwb+ZPdZUJfVmlpQeST7MgPHSedotFCNH49ex5H5MnT2X27BdQFBWTqYaEhIeIiYnlww/f5+DBA3h4uOPhoWPWrJcAcwO+119fxJ49u3n8cdnUYHMGg+HKjhNfVFUlIyOD2NhYSktL8fT0xMvLi5qaGnbt2kVsbKw9QgLAr28/Wj3YFwVqNeU7fkJ22Akhbi40tBU7d9Zuhjd06HCGDh1+3bnJyS/XeY3One/lvfc+skl8jY1dElJRURFJSUmYTCYURaF9+/a8+uqr/PLLLyxYsACNRkNNTQ3du3fnhRdesEdIAHjHxBJ0ZVnBUsPOX0d5yUXCOnS1WxxCCCHslJDCw8PZtm3bdcdbtGhBenq6PUK4qZKiy2i1GlDLUFVFdtgJIYSdOddWDQe6VGjAL6AJ5ZYadlLlWwgh7EkS0hUlRQbzhoaiPLRaN3wDWjg6JCGEcCmSkDBv3SwtqbDssPMNDEHr5ubosIQQwqVIQgJKSypQVSxty2WHnRBC2J8kJMwlgwB8fbUYykskIQkhhAO4dC27q64WVdWoV6p9y4YGIUQ9HNEPadGiVzh4MJNmzZpRUVFBYGAQI0eOrvVF2vz8PFJTV3Ly5Ak0Gg1hYW2YMWMmkZHtbv8mHUASElBcZMDXzxN9aQGAbPkWQtTLUf2QJk2azJgx46ipUThx4hgLFrxMaWkpY8dOoLq6mpkzn2P06HEsWbIcgM8+28XMmdPZtGkLTZs2rOanPUlCwrzl2z/Ym9KLx3H38MTbL9DRIQkh6nH853yO/ZRvtetd234ipktLou9t2eBr2asfUlRUDElJs1i2bAljx05g164MAgICGDt2guWcwYMf5vPPP2Pr1o9ITJzc4HuyF5dPSKqicqnYQKu2oZQWmTc0aDQaR4clhGiE7N0PqWPHzhQVFVJWVkpW1ik6dbr3unM6dbqX48ePWuP2bM7lE9KlkgpqahT8A705cTyXsA7dHB2SEOImou+9s1nMb1mrgrb9+yH92uXg6gyvMXP5hFR4wVwy3cdHocpokM+PhBANpqoqjz46ismTp1732MSJiQwY8CDff/8d//EfKTzwQD+mTPnDHT3f0aNHCA5ujp9fM+65pwM7d35y3TmHD/9MdLT9ilbfCZff9l1YoAdAi7kpn2z5FkI0VN++A/j00x0UFl4EwGQyceyYebns3LkzhIWFM3LkGMaMGc/Ro+Z+bN7ePuj1+tt+rpMnT5CaupInnjDPxn73u6GUlBTz0UebLed89tkufv75J0aNGnunt2YXMkMq0OPl7UFF+QVAEpIQouFs3Q/p3XffZvv2j6moMBIYGMjTTz/Lww8/Aph7za1cuZbU1FV8+OH7aDQQFhbO6tVp+Po2ji7DGvUuWHhsaMdYgPTNP6GYFFoGHyb/9BFG/HGJlaOzHlfuFmpLEqttWDvW/PyztGzZ1mrXu5YzdmG9EWeM9bfvTUM7xrr0kp2qqhQW6M1bvqVkkBBCOJRLL9lVGKoxVlTjH+jFyeN5tO/S19EhCSFc2L/+9SVvvfW3645Pn55Er159HBCRfbl0Qrpaw66JVyWmmmqZIQkhHKpfv4H06zfQ0WE4jEsv2ZUUXQZAe6WGXbMgqWEnhBCO4toJqdCAztMd4+VCAJoFW++LdkIIIW6PSyckRVEJjwygrCgXn2ZBuHt4OjokIYRwWS6dkPr/rgPjn47jUmGetJwQQggHc+mEpNVqAAV9yUXZ0CCEEA7m0rvsAEou5qGqCs1khiSEuEWOatC3e/enfPDBx7Ru3QaA7OxzPP74Ywwa9BALFy4FoLDwIuvWreHnn3/C19cXNzctI0aMZvjwEWzYsI709G0EBwdjNBpp2rQpQ4YMY9SosWi15vlJWVkpa9eu5sCBTNzc3AgICGD69CS6dLF94Wm7JaSEhAR0Oh2enubPaZKTk+nfvz8HDx5kwYIFVFZW0rp1a5YvX05QUJC9wqK44DwgTfmEELfOUQ36oqJiyMhIZ+rU5wDIyEgnKirG8rjBYOC556YyYsRjvPLKIjQaDeXl5ezd+7+Wcx555PdMn54EwPnz2SxcOJ/c3BySkmYBMHfubGJiOvLBB1vRarVkZn7P3LnJbNiwkVatWjf4Pm+FXWdIa9asISoqyvKzoijMnj2bpUuXEhcXx7p161ixYgVLly61W0xF+efRat3wDWhht+cUQtyZ00f2cfrQt1a7nkYDV4uoRXbuTWTH+xt8LVs26Bs06CF27vyEKVOmA/D553t49NFRHD9+BIDduzMICgpm4sRJlt/x9fVlxIjH6rxeWFg4L7/8Cs888yRTpvyRo0cPk5+fx+rV6ywzph494nj44WG89947vPjivAa/LrfCoUt2hw4dwtPTk7i4OAAmTJjAoEGD7JuQCnLwDQxB6+Zmt+cUQtydbN2gz8fHh5iYjvzwwz7AvER4beHU48eP33afpXbt7sHDw4Ps7HNkZZ0kJiYWd/faqaFTp3vZvPm927puQ9g1ISUnJ6OqKj179mTWrFnk5eXRqtWvn90EBgaiKAqXLl3C39/fLjEV5Z8nMDTSLs8lhLCOyI7339Es5rcaU4O+YcN+z86d6aiqytChwy2tLu7E1Rrbjq61bbeEtGnTJkJDQ6mqqmLJkiUsWrSIhx56yCrXbkhVWYBKowF9aTGde8fTvHnjKM/eWOIEidVWXDXWCxe0uLvbbmNwQ67t5qYFNJbf1Whg5MjHmDJl2nXnTpr0FPHx8ezfv4+VK1Po27c/U6f+EY0GtNqb39vV8+67735WrlyGyVTDa6/9hR07PkGjMccQGxvL7t2f3vBaWq0GVdXUevyXX7KoqakhMjKCigr9lX5KSq1Z0tGjh+jQoUM919Va5b22W0IKDTVvGtDpdEycOJHp06czadIkcnNzLecUFxej1Wpve3bU0PYTF3N+AcCjSVCjKOnvyq0HbElitQ1rx6oois3aLjR0hmQyKYBq+d0+ffqTkrKYYcNGEBzcHJPJxMmTJ4iJieXcuTO0aRPBo4+G4eGhY8+e3dTUKDRp4kNZWdlNn19Vza+BVqtlxoyZKIqKqmqu/GuOYfDgIfz97xt57713GT/+CTQaDXq9nr17/5dHHx2Foqgoyq/x5uScZ8mSRYwePQ6dzotu3eIICWlJWloq06cnWTY1ZGSks2HDxhvGqChKrfe6oe0n7JKQDAbDlQ/4fFFVlYyMDGJjY+ncuTNGo5Hvv/+euLg4Nm/ezJAhQ+wREgClRXmA7LATQliHrRv0XdWnT786j3t7e7Nu3ZusW7eGceNG0qSJF+7uHowZM95yTkZGOvv2fYPRaMTHpykPP/wIo0ePszy+ZMly1q5dxfjxI3FzcycgwJ+//nUFrVuH3eGrc3N2adCXnZ1NUlISJpMJRVFo37498+fPp0WLFmRmZvLqq6/W2vYdHBx8W9dv6Azphz0fcfbofkY9l4JGo7nt37c3V/7r2JYkVtuQBn224YyxWqtBn11mSOHh4Wzbtq3Ox3r06EF6ero9wriOm7sH4R06NYpkJIQQdzuXrtTQbeBIgoObUliod3QoQgghDfocHYCjyexICOEspEGfEEI0Eo7+noy4njXfE0lIQohGwd1dx+XLZZKUnIiqqly+XIa7u84q13P5JTshROMQENCckpKL6PWXrH5trVaLojjXzrUbcbZY3d11BAQ0t861rHIVIYSwMTc3d4Jt1LfMlbfTOxNZshNCCOEUJCEJIYRwCnfFkp25Fbnjft+eJFbbkFhtQ2K1DWePtaHx2aV0kBBCCHEzsmQnhBDCKUhCEkII4RQkIQkhhHAKkpCEEEI4BUlIQgghnIIkJCGEEE5BEpIQQginIAlJCCGEU5CEJIQQwincFaWDGuL06dPMmTOHS5cu4e/vT0pKChEREY4OC4CSkhJefPFFzp07h06no23btixatIjAwECio6OJiopCqzX/LbFs2TKio6MdGm9CQgI6nQ5PT08AkpOT6d+/PwcPHmTBggVUVlbSunVrli9fTlBQkMPiPH/+PM8995zl5/LycvR6Pfv27bvhPdhTSkoKu3btIicnh/T0dKKiooD6x6qjxnFdsdY3bgGHjd0bva71veeOGrt1xVrfuL3ZfTQ6qotKTExUt23bpqqqqm7btk1NTEx0cES/KikpUb/99lvLz6+//rr68ssvq6qqqlFRUaper3dUaHWKj49Xjx8/XuuYyWRSBw8erO7fv19VVVVNS0tT58yZ44jwbmjx4sXqwoULVVWt+x7sbf/+/Wpubu51sdQ3Vh01juuKtb5xq6qOG7s3el1v9J47cuzeKNZrXTtuVdU5xq61uOSSXVFREUeOHGH48OEADB8+nCNHjlBcXOzgyMz8/f3p1auX5edu3bqRm5vrwIhu36FDh/D09CQuLg6ACRMm8I9//MPBUf2qqqqK9PR0Ro8e7ehQLOLi4ggNrd3vp76x6shxXFeszjpu64q1Po4cuzeL1RnHrTW55JJdXl4eISEhuLm5AeDm5kaLFi3Iy8uzLC84C0VReP/990lISLAcS0xMxGQyMWDAAJKSktDprNM++E4kJyejqio9e/Zk1qxZ5OXl0apVK8vjgYGBKIpiWVpytL179xISEkKnTp0sx357D35+fg6M0Ky+saqqqtOO47rGLTjf2K3rPXfmsVvXuAXnHLsN4ZIzpMbkL3/5C97e3jz55JMAfPHFF3z88cds2rSJU6dOkZaW5uAIYdOmTXzyySds2bIFVVVZtGiRo0O6qS1bttT6K7Mx3oMz++24Becbu43xPf/tuIXGeR834pIJKTQ0lIKCAkwmEwAmk4kLFy7c1rTeHlJSUjh79iyrV6+2fBB8NcamTZsyduxYMjMzHRki8GtMOp2OiRMnkpmZSWhoaK3lmuLiYrRarcP/wgQoKChg//79/P73v7ccq+senEF9Y9VZx3Fd4xacb+ze6D131rFb17gF5x27DeGSCSkoKIjY2Fh27NgBwI4dO4iNjXX4Mse1Vq5cyaFDh0hLS7Msa5SWlmI0GgGoqalh165dxMbGOjJMDAYD5eXlAKiqSkZGBrGxsXTu3Bmj0cj3338PwObNmxkyZIgjQ7XYunUrAwcOJCAgALjxPTiD+saqM47jusYtON/Yre89d9ax+9txC849dhvCZRv0ZWVlMWfOHMrKyvDz8yMlJYV27do5OiwATp48yfDhw4mIiMDLywuAsLAwnn32WRYsWIBGo6Gmpobu3bszd+5cfHx8HBZrdnY2SUlJmEwmFEWhffv2zJ8/nxYtWpCZmcmrr75aa+tscHCww2K96uGHH2bevHkMGDAAqP8e7Gnx4sXs3r2bwsJCAgIC8Pf3Z+fOnfWOVUeN47piXb16dZ3jNi0tjQMHDjhs7NYV6/r16+t9zx01dm80BuD6cQvOM3atxWUTkhBCCOfikkt2QgghnI8kJCGEEE5BEpIQQginIAlJCCGEU5CEJIQQwilIQhKikTh//jzR0dHU1NQ4OhQhbEISkhBCCKcgCUkIIYRTkIQkxB0oKCggKSmJ3r17k5CQwLvvvgtAamoqzz//PDNnzqR79+6MGjWKY8eOWX4vKyuLxMRE4uLiGDZsGHv27LE8ZjQaef3114mPj6dnz548/vjjlrI7AOnp6Tz44IP06tWLN954w3L8p59+4rHHHqNHjx488MADLF261A6vgBBWZPcOTELcJUwmkzpq1Cg1NTVVraysVM+dO6cmJCSoX331lbpmzRq1Y8eO6qeffqpWVVWpb731lhofH69WVVWpVVVV6uDBg9U33nhDraysVP/973+r3bp1U7OyslRVVdXXXntNffLJJ9X8/Hy1pqZG/eGHH9TKyko1OztbjYqKUufNm6dWVFSoR48eVTt16qSeOnVKVVVVHTdunLp161ZVVVVVr9erBw4ccNhrI0RDyAxJiAb6+eefKS4uZsaMGeh0OsLDwxk3bhwZGRkAdOrUiSFDhuDh4cHkyZOpqqrixx9/5Mcff8RgMDBt2jR0Oh19+vQhPj6enTt3oigKW7ZsYd68eZZeRz169KhVqHTGjBl4eXkRExNDTEyMZebl7u7OuXPnKC4uxsfHh27dujnkdRGioVyyQZ8Q1pCTk8OFCxcsnUXB3AIiLi6OVq1a0bJlS8txrVZLSEgIFy5cAKBly5a1WjO0atWKgoICSkpKqKysJDw8/IbPe22RzyZNmmAwGABYsmQJa9asYejQoYSFhTFjxgzi4+Otdr9C2JokJCEaKDQ0lLCwMHbv3n3dY6mpqeTn51t+VhSFgoICSxXm/Px8FEWxJKW8vDwiIiIICAjA09OT7OxsYmJibiueiIgIVq5ciaIo7N69m+eff57vvvsOb2/vO7hLIexHluyEaKAuXbrg4+PDhg0bMBqNmEwmTpw4wU8//QTA4cOH2b17NzU1NWzcuBGdTkfXrl3p0qULXl5evPXWW1RXV/Pdd9+xd+9eHnnkEbRaLaNHj2bp0qWW5nsHDhygqqrqpvFs377d0kzuagvra2dhQjg7Ga1CNJCbmxvr16/n2LFjDBo0iN69ezN//nz0ej0AgwYNIiMjg/vuu4/t27eTmpqKh4cHOp2O9evX89VXX9G7d28WLlzIsmXLaN++PQAvvfQSUVFRjBkzhvvvv58VK1agKMpN4/nnP//JsGHD6N69O0uWLGHVqlWWvkRCNAbSD0kIG0hNTeXs2bOsWLHC0aEI0WjIDEkIIYRTkIQkhBDCKciSnRBCCKcgMyQhhBBOQRKSEEIIpyAJSQghhFOQhCSEEMIpSEISQgjhFCQhCSGEcAr/B7CkAfc2shJcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set()\n",
    "epochs = [10*i for i in range(20)]\n",
    "plt.plot(epochs, net_scores[:20],label='Net')\n",
    "plt.plot(epochs, net_scores[20:40], label='Net_DO')\n",
    "plt.plot(epochs, net_scores[40:], label='Net_MCDO')\n",
    "plt.plot(epochs, test_scores[:20],label='Test')\n",
    "plt.plot(epochs, test_scores[20:40], label='Test_DO')\n",
    "plt.plot(epochs, test_scores[40:], label='Test_MCDO')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('Test accuracy')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}