10. Computer Vision/03. VGG Net - MXNet.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.5-final"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __VGG Net  - MXNet (From Scratch)__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import nd, autograd, gluon\n",
    "from mxnet.gluon import nn\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "ctx = mx.gpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Getting the dataset__"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Now lets try a new type of dataset, known as Cifar10. It containes coloured images with shape of (3, 32, 32), note that number of channels is 3 as its a coloured image containing rgb values, and the number of classes are 10 in total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = gluon.data.vision.transforms.ToTensor() # this simply reshapes the image according to channel first format(NCHW), and scales it so that images's max and min value is 1 and 0 respectively.\n",
    "\n",
    "train_mnist = gluon.data.vision.CIFAR10(train = True).transform_first(transform)\n",
    "test_mnist = gluon.data.vision.CIFAR10(train = False).transform_first(transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "(3, 32, 32) 0.99607843 0.019607844\n9\n"
    }
   ],
   "source": [
    "image, label = train_mnist[1]\n",
    "print(image.shape, image.max().asscalar(), image.min().asscalar())\n",
    "print(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAfq0lEQVR4nO2da5BdV5Xf/+u++v1utdSSWmpJloRs2ZaMUGzsAIlnsCGkDDWBgg8Tf6BG8wEqoTL54GKqAvlGUoEpPiRUmeAaMyE8KsDgMkwGxxgM4xfySw/L1vvd3ZJaavXrvu/Kh76uks3+726r1bc1c/6/qq6+vVfvc/bd56xz7t3/s9Yyd4cQ4p8+qeUegBCiMcjZhUgIcnYhEoKcXYiEIGcXIiHI2YVICJnFdDazBwF8E0AawP9096/F/r+js8v7BlYGbaXCLO1XKRWC7e5G+2RzzdSWa+K2dDZHbalUeH+F/DTtUyrmqc2rVWoz8PeWSqd5v1T4+t3W3kH7NEXmw6sVasvn+TEDwpJuzWu0RyHP56oaGUdMPmamSoWPo1aLbY/3y2S4O2Uy/Jg5wudBTBWvkWHkZ/MoFkvBk+e6nd3M0gD+O4A/BnAWwO/N7Al3f4P16RtYib/8xv8I2s6++TLd18UTh4Lt1Sof/sp176O2dZu2UVvPqnXU1twS3t/hg8/RPqeO7qO28hS/SKQj762zp4vaMs2twfbd936I9rllC5+rwtXL1HbwwKvUVquVgu2lcvjCDQBvHNxPbZMTl6itWCpSW7kUdrLL4/xCNT3Lx1ip8n2tWNFLbT297dRW9anwvsq0Cwr58JXg18+8QPss5mP8bgBH3f24u5cA/ADAQ4vYnhBiCVmMs68BcOaav8/W24QQNyGLcfbQ94I/+GxhZnvMbK+Z7Z2avLqI3QkhFsNinP0sgKFr/l4L4Py7/8ndH3X3Xe6+q6OTf9cUQiwti3H23wPYbGYbzCwH4LMAnrgxwxJC3GiuezXe3Stm9kUAf4856e0xdz8Y61OtVjF5Jby629fNVzJ9RViu80wn7TO4biMfR40vc6ZqfJW2NhuWfwpXxmkfz/OV3TX9A9S2bugWahu6ZT21rV6zNtg+QCRPAMhmm6it0h1e3QeAobWreL9KeDW+UODy2sQVrk5cusRVgUxEZoWFV+N7+vh7bm7jY7w6eYXampq5O9WcS4fZTHgsk1cnaJ9SMbwa70yTwyJ1dnf/BYBfLGYbQojGoCfohEgIcnYhEoKcXYiEIGcXIiHI2YVICItajX/PuAPlsOxVKnI5bHY2LOMMb+FP507PzFBbLBijtz8SZJINXxs3b95C+3zw7l3UtmZlWCYDgK6uFdRWzvBoudbmsIyTiURQWSUS2TbD5bAiOZYA0NoSlux6urncuGnjrdR26NBb1Abj4ygWw1JqV2cP7RMJfMTVyTFqc4TPUyAeSXflSvhczc/yoBsWEReLANSdXYiEIGcXIiHI2YVICHJ2IRKCnF2IhNDQ1Xiv1VAhgRBW4SvMTbmWYPvVSzxVUd8qvtK97jYeZDIwtJrasmyZNpI/qFzhK/9vjvAAmtnjF/k2U3zV9639rwfbP7CNr3R/aPcHqC22ujsZyU9w+tQfRDsDAHLZSG7AHA9s6l/BlZfTZ47wbZI0XdN5rtZMTvLzKpPluQE7O3nQUCxfH0uvF8uT19QUPheND093diGSgpxdiIQgZxciIcjZhUgIcnYhEoKcXYiE0HDprTgbljzaW7gk09kbDgq5684dtM/Qxs3UNhUJ/Hjr+Blqm5wNyyfTEzxX2PgEl9dGRnk+s85IIAxSPEDiyR/+ONie/Qy/rn/4nvuoLZvlsuKqVVymhIflq4kr4eonAPDKq7x6TiaSJ6+tg0t2lWpYOixN82OWjtwCY1VfqlUuiY5f5nJeCmHJLlZOqrs7HLCVjpSZ0p1diIQgZxciIcjZhUgIcnYhEoKcXYiEIGcXIiEsSnozs5MApgBUAVTcnSdcA2ApQ1NTNmgrpztov3xLuJD9iUlepue1371EbZfHeV61c+d5jrFsOhxSlE3x6KQiKYMEAIUCtw2u4Ifmwugpausk0VBTE5O0z+ETJ/g4BvupLZvlYxwcCpeGWk3aAeD0KJc939rPbQODXKY8eZpIXmV+zGolbqtG8v8157g82JQJn/cAkC+Et9nZySXFDCkZZZH7943Q2f+FOxFVhRA3DfoYL0RCWKyzO4BfmtnLZrbnRgxICLE0LPZj/L3uft7MBgA8ZWZvuvuz1/5D/SKwBwC6e/ijhkKIpWVRd3Z3P1//fQHATwHsDvzPo+6+y913tbWHF9qEEEvPdTu7mbWZWcfbrwF8FMCBGzUwIcSNZTEf41cC+KnNZbjLAPjf7v5/Yx1SqQxaW1cGbRcmeCTa0TNh2eWNg/zakorIQtVIqan8FE9EmCYSW77IZa2JKW6bipRWOnn2ELW1tXCZcuumrWFDRAL8h9/+mtrWb9hAbVu28rJXfX3hqKymZn5cujq5dJWq8OSWM0V+z2IllPITPPquWuVJQptbuIQ2Pcm32RmJzGtqDkeqlUqxkmjhCMxajcuG1+3s7n4cwJ3X218I0VgkvQmREOTsQiQEObsQCUHOLkRCkLMLkRAamnAync6guzccRXX0zGHab+RkOCqrNcsTL16d4ckcpycvUJtFpIuJqbBUNpHnUk2GRPkBQP/KAWpr6QhLVwCwZpiLIENExjnx+vO0T9q4LFeu8iivi5d4Ms3bb98WbL9l80baZygSvdZ+905q2/fmaWorFsKJTIvZSNQbuExWcy4Rj46G69sBQK6Jy4pdPew84DJwPh+O+Kw5f1+6swuREOTsQiQEObsQCUHOLkRCkLMLkRAauhpfLM7g2LFwbrg3jx2l/c6PHAu2VyNBKx1dbdS2dfMwtW3ftp3aRi6GV0BPXeTjWLEqHPgDAOs38SCTjj6+Uj92he/PL4WVi9On+Ir1xUiJqm23UhP+eEt4xR0AZqbJajFf3IeXuCpw8AWuJmzeysuArVzTHWx/4aVng+0AMDrGg5fKZb4aX8jz8V+JlL1qaQ+PMbayPkPKqMUCYXRnFyIhyNmFSAhydiESgpxdiIQgZxciIcjZhUgIDZXeZqYn8cKzT4UHspLkTgOwadvtwfaWSJmebbdupratW9ZSW7UQDiQBAE+F5aQZ8II4mWw4EAMA0umw5AIA5QoPnJiZukxtXaWwNFSpOu1z+gIPGmpuP8f31dlDbRs3DQfbPXJ/yU+E86oBwJsvvkZtnufnwfYHHgy2334HD8jJ7+XS27GjJ6mttZVnT+7q7qO2ueppf8jkJD8uxWJ4rlzSmxBCzi5EQpCzC5EQ5OxCJAQ5uxAJQc4uREKYV3ozs8cAfALABXffXm/rBfBDAMMATgL4jLtznaBOuVTBhTNhmWrnnf+K9mtqCucm6+UqGQZX8zxilyOlf84c5bJWqRaWw1LGQ7nSGS6FVJ3n0EMlVr4qLAECgFfD+2vvCuf+A4DxaR5Fl8rx6MGaczlvrpp3qBPv0d7Mj9nw6iFqa07zcaQQzht4+3YecdjdzSXRJ/K/pLbREe4CawZWU1vVwjkMs5ESZpOTYXnwUDZcKg1Y2J39rwG8W6x8BMDT7r4ZwNP1v4UQNzHzOnu93vq7b3cPAXi8/vpxAJ+8weMSQtxgrvc7+0p3HwGA+m+eaUEIcVOw5I/LmtkeAHsAIJvlOdSFEEvL9d7Zx8xsEADqv2nVBXd/1N13ufuuTKahj+ILIa7hep39CQAP118/DOBnN2Y4QoilYiHS2/cBfARAv5mdBfAVAF8D8CMz+zyA0wA+vZCdpVIZtLb3Bm3ZiIozMRH+4NDUyyWS2QrXeAq8WhNaejqoralmZINcevPIDBfKPMqruYV3TEXKNdVS4X7tfVz6yTmXG9MtPLLNc1z7rFn4vVmVS3mpNH/P2bYctbW0c1ulGJZZx8+N0T59bbwM1UMff4Da9r5+ktqmI8koC8WLwfYiKfEEAN0d4XM/k+bHZF5nd/fPEdP98/UVQtw86Ak6IRKCnF2IhCBnFyIhyNmFSAhydiESQkOfcsnlmjC4LhxtZCl+3SkUwhE+Y5N8+LluHuVVrnCpxiJP+eWnwxFUZedjz2R44shKmttaO3kE2EDfBLX55bBcU4rUKLMaH39LSwu1pSJRhzUP769a5TJlKhtJ9pnmY5ye4VGMRhIwNkXOt8mLXJZraQ1LxwDwoXvuoLa3jp2itgNvjAbbpyd5NGKOJDKt1WIRgEKIRCBnFyIhyNmFSAhydiESgpxdiIQgZxciITRUenMD3MLySjkiDc1OhaWVpogsNDUZSRxZ4IkeZye5jJMlQW8dbVxCW9HDpZrOXh4BtqKbv7dqpova8k3heby8nke9Fasj1IZIZF61Eom+IxGC1RSPRrSI9Nbdy6PvatXIGMl51dXF5zdnXL6amIrInuWwNAsAO7atorbujvD58+STPLnlxbFw4tZKxI90ZxciIcjZhUgIcnYhEoKcXYiEIGcXIiE0Nt2rO0BWcDM1vrLbFX7mH0NdZHkcwPs28vx07c18JTZt/Po3MxleiS3MXqV9WtrK1LZ1M1+pH1q/ltpS2fXUNj0RHuPQ4CAfxwmaHBidvWTyAfT28GCdTCYcbBSJ04BHAmua21qprVKIrECT/WVjgVfgak1ffzu1Tc9yVWBmIhzsAgBrVoRz3n3yX3+U9vnbn/+/YHsmwydRd3YhEoKcXYiEIGcXIiHI2YVICHJ2IRKCnF2IhLCQ8k+PAfgEgAvuvr3e9lUAfwbg7bo1X3b3X8y3rY62Vnz4nvcHbRtvvZP2O3/uXLB9zWouXW3ZvInaVq3gFabTzuW8KRIEUYwEi1iKb6+9jQfCtLdzySud49JhlkiY+ZlwiSEAuGs7l/KGtwxTW7nGZUUn95FKjctknuZzlc7yU7Vc4HpejQSGpDL8PmfNfByI9CuW+Xxk0jy3YbUUPq9WRGS++/75B4Ltz7+0n/ZZyJ39rwE8GGj/K3ffUf+Z19GFEMvLvM7u7s8C4PGiQoh/FCzmO/sXzWyfmT1mZjzYWAhxU3C9zv4tAJsA7AAwAuDr7B/NbI+Z7TWzvdMzPLhfCLG0XJezu/uYu1fdvQbg2wB2R/73UXff5e672tv4goMQYmm5Lmc3s2ujKj4F4MCNGY4QYqlYiPT2fQAfAdBvZmcBfAXAR8xsBwAHcBLAny9kZ62tLXj/He8L2m7byaW3/PawjNbWxaOueKYzwI1LK6mIRNLbFs4jFqn+FL2a1khpIiCeSwwRiadYDJd/2nTLOtqnJcclwPwMj+jzVOT0sbDNI/ndas5t1cgxi5U8KuXD81Gt8fecykTOj8gRnRrnEuypE2eo7d77dgbbZ8s8H2IrkQcjSu/8zu7unws0f2e+fkKImws9QSdEQpCzC5EQ5OxCJAQ5uxAJQc4uREJoaMLJVCqFFhLp1d7MSyi1tZJhRpLrxRIbWkx6i0k8HpbKamUuocXkJIskPaxExMOYvOIkYWZ7N48QrFT5vqq1SBZIUuIJABzVYHsqNvgqt1UzXBJ1RA42SXBqtfD4AKAp8p6zVX7M2gq8n4+FJUAAuHh8LNi+ditPOnopFX4aNTa9urMLkRDk7EIkBDm7EAlBzi5EQpCzC5EQ5OxCJISGSm/pdBodXWEJyCPRZrPFsHziRV6Tq0j6AMDM9Ay1lcq8X7EYjjarVLh0VY5EqJUj+5qN1A2bneHRUBUSSdfR20X7dHTxunjdHf3U1pwL13MDgCqr3WeRumzgto4OnoBz/AKfx0I+LFHVajy5koG/r1qVn3OdHVw+Xr9uJbXlZ8Pno0eSc3Z1hCXsdETO1Z1diIQgZxciIcjZhUgIcnYhEoKcXYiE0NDV+ImJSfztE38XtFWzv6X9rlwJBwpMX71E+6QisRGxlfqxsfC+AKBKomt6I+Wkevr7qK0pzad/5nK4JBAAHD5yiNomp8Orz0MbeImndJYrIZ0dfPwbNvC8dmuHwvn6NmxcQ/v0NvEojo5mPsZaJBch0uHglHKVr3SnIyWe0pExrhyOKBedfKW+7OGgnDQXBdDbG37PmUhwmO7sQiQEObsQCUHOLkRCkLMLkRDk7EIkBDm7EAlhIeWfhgB8F8AqzFVVetTdv2lmvQB+CGAYcyWgPuPuV2LbmpyaxlPPPBe0da/dSvt5NSwnvfrcM7TP+rU8f1d/H5eTzp0dpbYKyVvW2ssDSUopHiQzdpaXBLp/9z3UtuOO26httlgItqey/FCfOH2K2g4fOUZt+w+8Sm3dXeEinn/ybz5F+9x72xZqy0VqbK0dHKK2EpHeLJKsLZY3sExy6wFAKhPJa9fNA3laSPBKLc0lYiZERlIoLujOXgHwF+6+DcDdAL5gZrcCeATA0+6+GcDT9b+FEDcp8zq7u4+4+yv111MADgFYA+AhAI/X/+1xAJ9cqkEKIRbPe/rObmbDAHYCeBHASncfAeYuCAD4Y2RCiGVnwc5uZu0AfgzgS+4++R767TGzvWa2t1Tigf9CiKVlQc5uZlnMOfr33P0n9eYxMxus2wcBXAj1dfdH3X2Xu+/K5fjzwUKIpWVeZ7e58infAXDI3b9xjekJAA/XXz8M4Gc3fnhCiBvFQqLe7gXwpwD2m9lr9bYvA/gagB+Z2ecBnAbw6fk21NPbh09/7t8GbU0Dm2m/2amwHHZk/+u0z+AqLsekInm6Wpp5BFWpFi7hs2U7H3vPIF/KmO3nedA+8bE/orbWjhZqmyHSW6RSEyqkrBUAFCrh7QHAhQuXqe3UifPB9tZWPr+jZ8ep7eTBI9SWKvAxHh8NfuDE7o/uon3WD6+mtli0XKo5EqaW5bKcsVxzxvvkLHzMYtLbvM7u7r8DwDZx/3z9hRA3B3qCToiEIGcXIiHI2YVICHJ2IRKCnF2IhNDQhJNmQFMufH05/OYB2m/yalh681h0UolHDE1Hyj9ZRLtobgrHGpVneTmmqxf5GMdO86i3v/v7cGJOALgyFdnf9NVge0cnl7y6esIluQCgLZIo8ezZsLwGAAP94cSSzZ1civztz/l7vnxkH7VVS7zE1tHRcALRs5ESWpu3cSm1q7OV23p4ia2WVh711tUWPq+yzTx5ZGtr+Li48/NXd3YhEoKcXYiEIGcXIiHI2YVICHJ2IRKCnF2IhNBQ6a1WKWNqPCyj/epnP6f9zoyeDbanyuEoNADYty+SXyMir1UqPKoJJNLoqSd/Rbvksly62rHzLmor5TqobbI4S23HT4ejvMbHeX24UoFHvZ0fPUltJ07ybe7a+f5g+7/7wn+gfV564Xlqq1zlEXGTRZ4UJY+w9Hl8L5c9f/vyCLW1ZbjMl81xqSzdxM+DDiK9rV0/TPs89CefDbaXKvz+rTu7EAlBzi5EQpCzC5EQ5OxCJAQ5uxAJoaGr8dlsDoMrB4O2zcMbaD9HeLU4EymtlI6suKfS/BrnNR64kmtuCxuyPMhh9epwQAgAfOSBB6itozUScNHMc9e9cSCcl+/wUV7GadWaYWorRMoupVv4GA8cfjPY/sbhw7RP6/A2ajt/nr/nnm5uG8iF88K1tvM8fpdHeTms8XNHqe3ipXDQDQAUqpGgLZIgcGSCu+cH7w/3qfC0dbqzC5EU5OxCJAQ5uxAJQc4uREKQswuREOTsQiSEeaU3MxsC8F0AqwDUADzq7t80s68C+DMAF+v/+mV3/0VsW5VKBZcvhksG3f3PPkj7ffDDHw62NzXxwINMRF6LlX+qRUohpRHeX7nE9Y58iQetjJ89QW2XCzzg4vIlXnbpOJHYzl8IByABQPsAL3eEJi4rWo5Lb6VKODjlqd/8jvZZv+l2ahvq5RJmc4qfxq0kEKlY4Dnojk8epLb2Dp7Lr+o8iGr0yjS19fcPB9tny/xc/NVvXgq2T03x/IoL0dkrAP7C3V8xsw4AL5vZU3XbX7n7f1vANoQQy8xCar2NABipv54ys0MA+GVWCHFT8p6+s5vZMICdAF6sN33RzPaZ2WNmxh9jEkIsOwt2djNrB/BjAF9y90kA3wKwCcAOzN35v0767TGzvWa2d2qaf08SQiwtC3J2M8tiztG/5+4/AQB3H3P3qrvXAHwbwO5QX3d/1N13ufuujnaefUUIsbTM6+w2VyLlOwAOufs3rmm/NqLlUwB4SRchxLKzkNX4ewH8KYD9ZvZave3LAD5nZjsAOICTAP58vg2lUoY2UrZmfLJA+7267+Vg+8AAXyZYOdBPbeUyl7WuXJmgNhTCY8zU+PbWbOCy1lAP/6Rz7jDPgzYzzXOuDaxcFWxv7eumfdLNXE6azfPjMji4jtpGz4fzBl4aD5enAoDB1ZGyXJFSX9NFPv/IhM+3co3LpU0tJLoRQFMkmrI0fpHakArnmQOAlSTqsFTkJczYdPBZWthq/O8AhN5hVFMXQtxc6Ak6IRKCnF2IhCBnFyIhyNmFSAhydiESQkMTTqYMaMqGI3mKBS55Pffc08F2L3NZqLOVJxQsl3l0UiHPS0plyLVx/fAQ7bP97lupbdM6LstNnAlLVwAweuUSteVawlLTpr6wJAcAFy/yiKzbt26ntttu30ptP/hf3w22ZxBOAAkA5Rl+PEslbvNYlsXm8LGOlWMa3rCR2i6ceYvvK8WjMFva+P62bdsSbC/M8uMyNDgQbP9Njkt8urMLkRDk7EIkBDm7EAlBzi5EQpCzC5EQ5OxCJISGSm+1Wg2zeZKAMZIE8oGPfSK8vRKPkkpH5LValSfy8zSXT9KZsGzU3MYTL45OcClvaoLXPbuc5+O3Zp4E8q3Xjgfbx5/nEVkbN3AJ7QO3bKa2UiQiriUXlpo8EnEYi7BLpfmpSkqlAQDyNVInsMrnd/1aLr0Vpsep7dZOHi330suvUtv5U2E5Lz/Dz2+fvRJsLxV5RKTu7EIkBDm7EAlBzi5EQpCzC5EQ5OxCJAQ5uxAJobFRbylDW3tYvuqKZMrrWBGOCipGZIbmyHUsZzzyylt4tFxTa7hfrcCjk6amJqkt3coTPQ5s4gkiN7XyqLcjJ8K13mBcUsySJKAAcG7kNLX19fOEn8xWynM5qVjkyShnIhFxxUh0WLkYlnozzVwuXbl6BbWdGhmjtrHTZO4BFKb5ezt28LVge18fH4f39IbbI4k5dWcXIiHI2YVICHJ2IRKCnF2IhCBnFyIhzLsab2bNAJ4F0FT////j7l8xs14APwQwjLnyT59x9/DT+XVqtQJmp0jwR41fd7LWHmwfG+MrnEfeOEltzRm+4p7r4qvg/aTc1Or+LtonEwnw6evqo7ZIrA4KeT7NAwPhFf41q8OrtwAwMjpKbYcPH6K24dIGamNKydQUP2azs3yle/IqVzViq/HVUjgQKd3Eg1YOHuClw2IlmQYGVlLbmjt4Lr+BFeF+/St43sBmMv6n/+EZ2mchd/YigH/p7ndirjzzg2Z2N4BHADzt7psBPF3/WwhxkzKvs/scb186s/UfB/AQgMfr7Y8D+OSSjFAIcUNYaH32dL2C6wUAT7n7iwBWuvsIANR/h3PbCiFuChbk7O5edfcdANYC2G1m/AvIuzCzPWa218z2Tk2RxBVCiCXnPa3Gu/sEgF8DeBDAmJkNAkD99wXS51F33+Xuuzo6+COKQoilZV5nN7MVZtZdf90C4I8AvAngCQAP1//tYQA/W6pBCiEWz0ICYQYBPG5macxdHH7k7k+a2fMAfmRmnwdwGsCn591SzVEjZXxSketOphwO4ugkpaQA4OUXfkNto2M8kMSyPChk9+73B9vvu2cX7XP1Kpea9r3yIrXNFHjgx+HTZ6jt+MmTwfb8LP8K5c6TuDV38mCMyckpapsiJapmJrlsGEklh0yaW7sinxhXbwjLgz19g7TPwGouea3eeTu19UZy0OViuQ2ZLRK8BA/7SypSgmpeZ3f3fQB2BtrHAdw/X38hxM2BnqATIiHI2YVICHJ2IRKCnF2IhCBnFyIhWCxn1Q3fmdlFAKfqf/YD4BpY49A43onG8U7+sY1jvbsH9dKGOvs7dmy21925QK1xaBwaxw0dhz7GC5EQ5OxCJITldPZHl3Hf16JxvBON4538kxnHsn1nF0I0Fn2MFyIhLIuzm9mDZvaWmR01s2XLXWdmJ81sv5m9ZmZ7G7jfx8zsgpkduKat18yeMrMj9d+8ttLSjuOrZnauPievmdnHGzCOITN7xswOmdlBM/v39faGzklkHA2dEzNrNrOXzOz1+jj+c719cfPh7g39AZAGcAzARgA5AK8DuLXR46iP5SSA/mXY74cA3AXgwDVt/xXAI/XXjwD4L8s0jq8C+I8Nno9BAHfVX3cAOAzg1kbPSWQcDZ0TzEX7ttdfZwG8CODuxc7HctzZdwM46u7H3b0E4AeYS16ZGNz9WQCX39Xc8ASeZBwNx91H3P2V+uspAIcArEGD5yQyjobic9zwJK/L4exrAFybfeEslmFC6ziAX5rZy2a2Z5nG8DY3UwLPL5rZvvrH/CX/OnEtZjaMufwJy5rU9F3jABo8J0uR5HU5nD2UcmS5JIF73f0uAB8D8AUz+9AyjeNm4lsANmGuRsAIgK83asdm1g7gxwC+5O68KkTjx9HwOfFFJHllLIeznwUwdM3fawGcX4ZxwN3P139fAPBTzH3FWC4WlMBzqXH3sfqJVgPwbTRoTswsizkH+567/6Te3PA5CY1jueakvu/3nOSVsRzO/nsAm81sg5nlAHwWc8krG4qZtZlZx9uvAXwUwIF4ryXlpkjg+fbJVOdTaMCcmJkB+A6AQ+7+jWtMDZ0TNo5Gz8mSJXlt1Arju1YbP465lc5jAP5ymcawEXNKwOsADjZyHAC+j7mPg2XMfdL5PIA+zJXROlL/3btM4/gbAPsB7KufXIMNGMd9mPsqtw/Aa/Wfjzd6TiLjaOicALgDwKv1/R0A8J/q7YuaDz1BJ0RC0BN0QiQEObsQCUHOLkRCkLMLkRDk7EIkBDm7EAlBzi5EQpCzC5EQ/j8Td47fuM9PmAAAAABJRU5ErkJggg==\n",
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n<svg height=\"248.919844pt\" version=\"1.1\" viewBox=\"0 0 251.565 248.919844\" width=\"251.565pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n <defs>\r\n  <style type=\"text/css\">\r\n*{stroke-linecap:butt;stroke-linejoin:round;}\r\n  </style>\r\n </defs>\r\n <g id=\"figure_1\">\r\n  <g id=\"patch_1\">\r\n   <path d=\"M 0 248.919844 \r\nL 251.565 248.919844 \r\nL 251.565 0 \r\nL 0 0 \r\nz\r\n\" style=\"fill:none;\"/>\r\n  </g>\r\n  <g id=\"axes_1\">\r\n   <g id=\"patch_2\">\r\n    <path d=\"M 26.925 225.041719 \r\nL 244.365 225.041719 \r\nL 244.365 7.601719 \r\nL 26.925 7.601719 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n   </g>\r\n   <g clip-path=\"url(#pf981f26bff)\">\r\n    <image height=\"218\" id=\"image5ec7e44540\" transform=\"scale(1 -1)translate(0 -218)\" width=\"218\" x=\"26.925\" xlink:href=\"data:image/png;base64,\r\niVBORw0KGgoAAAANSUhEUgAAANoAAADaCAYAAADAHVzbAAAABHNCSVQICAgIfAhkiAAAFPxJREFUeJztncmPXNd1xs+bauyqnkf2SFLdpElapKXQlGRZlgNFMAJkgLMKkP8jf0g2yTJeZJFkoURJkMBxYluIAkmMJIoSqebUJHuonmueXr33ssj2fHcRGMeb77e8H27Vq1f11QPOPfe73s/+4s8zAcTiI0miQlkdPzvuwzl/9Zc/g1rt8ARqXpSH2u3br6njP3jjdTin0WhArdfD19/pY237xUuoPd3Z0d+r24VzssyDWqE6C7UwyEGtda7f407zHM7BVyESBlgdr5SgtjQ/o45PTi/COXNLFx2vh+/HVFX/nYqI5IIAagHSPDxHMuwXrBBCfmPQaIQYQKMRYgCNRogBNBohBtBohBgQ+rkCFNMU+3AUFdXxZjyEc1678w7UCqH+eiIiufEJqM3MTerjM+NwztwS/lxLS7jEnKRQkk/u3cfv9/ChOh6FIZxzUKtB7eSsCbX19Q2onR8fquOtximc0+3iJY1mA19Hp9uG2sNvn6njQf4IzrkheLngwYOvoRZl+Pd4YQEvC8zNzqvjM7MLcE4hj5cS+EQjxAAajRADaDRCDKDRCDGARiPEABqNEAPCUmUTir0WLo3GmV5unXB0Ur89j98r5+Gu86yIS//5kj4v7ePycquFy9KnjR7UPA//LxWK+jKDiMjRkf5+x0e4nD0zOwe1zc2rUJuewdfRbdbVca+Cl0ImJ/Fyx8QELv0PHOX9eKDvWggLuIR/7foNqH352X9B7dCxq+L5022oTc/o9396Gv++11fxDgM+0QgxgEYjxAAajRADaDRCDKDRCDEg7LRxZbHRwFWl/ZNddTwdduCcIB5BLXV07GaubIdQrzoWyriCtbjhaBwe4ork2a7elCsi8mQX524cHx6o46fHx3DOeBlXAi8srkKtPIabxD850atzWQxjY2ThAr5XxdI01NIEf9e5SG+mDnMRnDNRqUBtbVFvABYRWZ2rQu2Tu59D7dK1m+p4r4N/316Ef6d8ohFiAI1GiAE0GiEG0GiEGECjEWIAjUaIAWGpiMvgcoYjq//1Xz5Ux7MYLwlUS7g5OHaU/vs93Ogbgv+KtfUVOOfWrT+D2vXVJajdPcXZGl854sK3LuvNppPf1+PMRUSOj3FT7n4NLzNcu7EFtd5woI7nApx1USri5YLhEH9mL0ugVvT1JZkgwBkqtV3cHNwEUeciIi9fPIVa4MjLWVrT72Pf0Sy9sogbwflEI8QAGo0QA2g0Qgyg0QgxgEYjxAAajRADwkGMvZYv4CjuN9/8XXV8DkR0i4jMz+knPYqIxHEMtfNzPetCRERAWT1MHa93iF/vyRCXpTs9vMNgYRJ/ttqx3r3/BJwEKiISFHDX+fa3OH78q4ef4es41a9jaQ7vBojKuASeK+GTWLMBvv9joT4vTvG933mGy/RFD7+XOF6z19WXO0REHjzQ80SGA7zbZdDHS1R8ohFiAI1GiAE0GiEG0GiEGECjEWIAjUaIAWHHUeIMwKmeIiKv3b6jjufzOKAkDLCvfR9rc0u4Ez8Q/f1iR5m+N8S7Ep6dOyLBKziopjw4g9rRoX565zc7j+Gczevfg9oowd/Z8eELqKVg3sw0DgLKFXFnf5rhbvtCGWulSC/vD/otOGfwFIfi+IK/z5wjMEdS/F0f7u2o43GMl3g21pahxicaIQbQaIQYQKMRYgCNRogBNBohBtBohBgQTs1OQfGDf/p3qG3vPNFf0Mflz8DzoOY7Sv9ZirPhcwVQfo4cwStLF6D2/vvvQ61SwkFGkwd7ULv46JI6XncEvYxF+ATU5gCH4mSOpYucr9//9975AZzz9Zf4VMz9fX3ZQkRkfALv4ohy+mcrjeHlpInqGNRO9/B1nDsClbIE/65O9r5Vxxcv4J0OP37nNtT4RCPEABqNEANoNEIMoNEIMYBGI8QAGo0QA8IDcOyriMijnWdQ263tq+N+jDuiSyE+OlUcpf/RCIeeiKcvJ1QmcNd54xB3zecFd5DfvIk76g/2cR5+vqh3q29e1sv+IiLDPl4mKYDPLCKS9HB5/9YtPev/O5ubcM6nH3wAtbHGEdTOd/EOg57oOyu6YCeGiEjm4d0A5RCH88zO4JCjII/DhSpl/be67OjQn53AywV8ohFiAI1GiAE0GiEG0GiEGECjEWJAWJlegOKP//BPodZs6I2cv/zHv4NzrlzUT74UEWm3cSZEq4UrgR6oON1593fgnLyjQfXwGT5ZMjfE11HN44ySi6v6SZCvXr8M54xP4mbvchVXy3Z39WqwiMjcjN5MXa7Owjm377wBtbNHH0EtGeJK4GNwYunSVZwN88rVV6A2XsXN3uOTOA+lWMKN5+Og6hgVcGW0VMLN3nyiEWIAjUaIATQaIQbQaIQYQKMRYgCNRogB4WCIG1Q3r1yHWreln3D5qw//Hs5BWREiItMzuNTabDahNgAnS0alCpwzPquX20VEogQ3r/7k/Z9ArVTBeRcdkPGR4j5qGWX4e+mPcBl5ZQ1Hkz9/ppf+Xa/39nvvQm1nEn8vPjiJVURkqqYvJ9z+vdfhnLX1JajFCW46zxfwUkia4CUZL9VfM/Xw5xIPN1LziUaIATQaIQbQaIQYQKMRYgCNRogBNBohBoR/+zd/DcWJ5S2oZYkeZ91u4fLyQQ13xs9MT0Ot18dl5FGql2i37z+Cc6ZP8eud7+Hsjw/zP4ea+DgvogvK+36ElxKevXgOte1Hehy7iMjJ6QnUJsb1XQs//ZM/hnOuXsN5IoUe7qjPp7h7fyu4po4Hk3i9o+c4DdR3PC9Swa+ZOE7vRCfQpgHu3h85roNPNEIMoNEIMYBGI8QAGo0QA2g0Qgyg0QgxIHzv3TehmEQ4xOb8XC+Dz72Pu70dFXAZDIZQu7CMA4QScBpoIcA7BXIp/n9ZXcYBMV/cvw+17UcPoNZs60shKxtrcE4Q4fj0xUV8jW+++RbUllf0+7hxEZ+AmjlS3IeOaPLjA7yUIzl9WWNhHO+qCEL8fUaOKPEowVq3juPT40xfNipO4d0AeEGDTzRCTKDRCDGARiPEABqNEANoNEIMoNEIMSD8oz/AgTPrF3F3dnegZ+VnGQ5KcZXwO47s/WGM56FwntEIl57jGBdiY8d7dbu4HNztvAO1UQpOJZ3CufCV8QmoTVT0YCQRkYIjAClJwWfz8Hc2atWh1urj+3jewDskquBzRwHezSAZ1rIhfl4c1vBuhv3n+LTbrKh3/V8ew6e0nrUdYUVQIYT8xqDRCDGARiPEABqNEANoNEIMCFsNnPHhJbiqVMqBZk1H82exhE9mnJyahJrv4dyHNNObitMEVx0FzBER8UBWhIjI0HE/cgGed3KuV746/R6cM3Jcv+fI4xjGOOYafWqUjyEikgqu9rVaOB7b8/HvoFDUm9V9x5wkxc3BfoAbfZstHNP9/AXOh9m6pVcXPR/fj0arATU+0QgxgEYjxAAajRADaDRCDKDRCDGARiPEgLDXwc287T4ujfZ6eg5GebwK5zgK7s4Sfi7AwRVBqP9X+NH/L545BQ3AIiJh4njNDGseOL2zXcdLK8VcGWpBHt+rzHccI+rppenMc4S5BFgLRniZYeSI4kZLQJmPS/gDHy9bJCH+zjoFx6me8/iU1tmL8+r4ucMTXccRrnyiEWIAjUaIATQaIQbQaIQYQKMRYgCNRogB4d17D6F4PsJd0ft7e+r4haUpOGfzFZy3sDCL46DTEc7xaJzomRaDGOd7eI4S+FgZl9XHxvDSRRDhe5XP62XkJ49fwDkLc6tQW9+chVrs6Oz3PL18niS4BJ46ll0Cxy4I33GPc0X9fgSOzvh0iHNNUkd5vzKNd4zcnLsOte5I/2xeroLn9PUlLxE+0QgxgUYjxAAajRADaDRCDKDRCDGARiPEgPCXH9+F4tf7uFx5Xj9Xx9dW9K5nEZFaHYfRjBVwJ3Xg4f+DTlMv7/e7OCilWMYl362rV6C2sobDY/wIl9XbTX2poVjGZfr/uf8cao/3cajMq6/iGPepCX15InKU1eMEl/CTGJfcPUf3vh/p75c6YtyzvmOHQR7PyztOTo1BCV9EJFfUI9kPT3Ds90e//hRqfKIRYgCNRogBNBohBtBohBhAoxFiAI1GiAGhKyt/5MhCb4DY9ZcNXDJtP8WnRw77uGTdbbagFoEqcqWMu+lnF/FJmyePcGDObBOXrJN2DWq9k1N1/Kt7j+GcB0/xaZQL8/j6V1bx8kp1rKCO5328tOLhxn7pd/AOiTTBOy5Qhk3sCEbyHWcAnJ7gZagsxPOqU3h5ZWdXX7768MN/g3OOT5i9T8hvFRqNEANoNEIMoNEIMYBGI8SA0MtwWSkCzZ8iIqWKnp3gBbhSWaniPJE4h6OWx8fwaaC9tl5xGjjiqmvnuFrW8PD1D/K4Gbl7iCtO2ZleNX36fB/OOWziStrC8iLUAkcV2ff1inDg+L/NHCeI1s/0ypyISKHkyBoBv6t2Dzedt1u4gTkq4d+Vl9ebg0VEvniAG7fvf/OVOj5M8f2YnZ+BGp9ohBhAoxFiAI1GiAE0GiEG0GiEGECjEWJAePDiGRSnN3BkcqGgR2fPV8fgnBlHtkPfEfs96OBSfT7SY65bCV4uyEZYCx3zuk2cF3F0CrqsRaTS10vTOcfySebje9VzlMEd1WfxwYmfgeOkzdRR3vcSfI1jZVxWz3z9/33oeL3qLG6WPmvia/zs43tQa/fwb84L9XsyVnWcxOrIXuETjRADaDRCDKDRCDGARiPEABqNEANoNEIMCLttnJER4wZsmZjQT+gsO+aUQuzrSI+zEBGR+i7ODBmmejZIK8Ol21wZZ2RMRbhDPxCcQ5Jmjq75VC/vt09x9/6w04Fa0sMnj3pDXOr2M/2zZYHjpM0Ed83HHXyPe22shXl950dlcgHOeQkyPERE/uGfcY5H7QDPW13bgFri6Z/bC/BvuA7i6UX4RCPEBBqNEANoNEIMoNEIMYBGI8QAGo0QA8K5FRwoUqvhyOo03tHnDB2nNib4NMqtzWWojV9fhdqLA72TvXGIy8vJCP+/BB4u4Ueh41TSvCNWe6i/X7txAuf4jnChdIhL/77nWF9Bp3A6/m7bfbxjYWf/JdS2ZnCgUir6Do+v7j+Fcz757Auo7e3hZZJchL+XoyM8rzCm/w6Oj4/hnI5jSYZPNEIMoNEIMYBGI8QAGo0QA2g0Qgyg0QgxILzzw/eg+PGXuLz/bPuBOp60cInz6bOvofZoG+fJX7+KQ4Jqx3p5/+QYX8fsAg56SRJ8mmbJkWtfruBlAW+gd8eHAS7Fr07h8vjG6gWoNZq4W317W79XN757Dc4pTuDdDFe+fxNqF7ewdniqL718dQ+X9xvneAfHeBV/Z6eneHdK70w/iVVEZP3yZXW8WsXfy2iET7vlE40QA2g0Qgyg0QgxgEYjxAAajRADaDRCDAgvXboNxfM2zk8ftPSSdWkVZ9c3Orj0/O3THahtP8Zl33pLP4K23sNZ+GE+gtrMvB46JCKysHoRaivrr2KtoOe4r67hXQnrG7iEPz2/BLXDoxrUbty4qo47ovfFy+EljWt33oDavYcvoLa9p2uDFg4WKpVwIFGa4QChThcfUVyd0EOCRERykf654yHeVVEu4Vx+PtEIMYBGI8QAGo0QA2g0Qgyg0QgxIKyf4dyKagVXehbX9TjlZIQbNasRbhxOBjhPpOdoVF4DJ2P2Bjjrot7CWquDq1S9VgNqe48/h9oA5J2vg3soIrKwhBuf1zbwvM2tTahNT+vNt2GIm5tfHuCMjM/+G3/mVoL/ww/O9Obm3jGumCaJo6l4ElcPKxVcCQwjXH3udLrq+HCIK5x5R24Mn2iEGECjEWIAjUaIATQaIQbQaIQYQKMRYkDY7R5CcayMm4q3tvRS/dLCd+CcRhs3+p6d4rL63j6+xgjkbkSg7C8iMhjhuPBuH2sbl3BZ/aj2BGp3P/pYHV9fxs3Bb739I6gtLuIY9yjCp3cGoIyfOrIuGk3cJJ6GOKujWMal82JdX1KKHE2+3Q5u5vUcMejlCn7No6MjqPX6+jOoWsVLXqUSzlfhE40QA2g0Qgyg0QgxgEYjxAAajRADaDRCDAgHA1w2HfNxx7QM9O7mjSouWa/c/C7UWiPcFb29uwu1Zle/jna9Duec1vEOg4MazjXpODq3yzMLUGsO9CWDygQuFW86OvTjEV4KyQTnbhy81MvqdUfc9s7jA/xeAb7+owOcGZLz9eWEUYT/9/2c45TWPA49GaX4fgwcp6qGoV6qbzl2fuRy+Dr4RCPEABqNEANoNEIMoNEIMYBGI8QAGo0QA8K8I8Z4FOshKiIizTM9tOWTT+/COU9e4i781ctrUNu6uAK1CEQ3i6N0G4/wLoLaAS79d7t4R0Ds467/Sf+n6vjNq1fgnCzBXfNxjLUTxymW9boecpTL6eFBIiLfu4WXZH710c+h1nGUwYNA7+wPfLzzIMG3Xo6P8amey8s4EGp6Cu+CaLf073MwwPe+XsfhTXyiEWIAjUaIATQaIQbQaIQYQKMRYgCNRogBYVjAeeFZhkvWg6Fe+l+cwSXTlmOnwOdf3IfapuP0zgLo+B60cBf+7DTuOr+ysAy18fFZqMUh7hL//bdeV8cjnIkjforFfozDaKpVHJizMK+XugsFHCrzi1/8J9ROjvegViricJ5WU99xMV6dhHMCD/+uXoJTX0VEmuC9RETyjpNfg1D/7YchfjaNHDtQ+EQjxAAajRADaDRCDKDRCDGARiPEgFAcpx7mUqyVSnoz72ENV6KW1q9CzQ9wdDPKb/g/9ErPo0fbcMav/wPHd89PTEFtdeUy1FYcTdFLF/RK5twcPtUzivJQK5bHoJZLcOVrBKLQz+s4GvvJ02+gFse42heEuGqKqn3NJs4u6fVxg7sIaCwXEc+h+Y4m5slJvTKdpri7eTjAn5lPNEIMoNEIMYBGI8QAGo0QA2g0Qgyg0QgxIKxO4nL27sNnUDs51vM/kgRnVng+LlmvXsKl/9R3nKRY1Eu0hclpfB2neAli7wSXumuHOJvi4ZNvoRaCpt3bb/0Qzrm8ifNE+g18HV/f/xxqaaqX94cxbtre38e/gVzoOFV16Mg8GerR2eenuAG43cXXOEocze9lvNxRKuGslHikLzWUSrgJ3/dY3ifktwqNRogBNBohBtBohBhAoxFiAI1GiAH/C95/5IoXnsCwAAAAAElFTkSuQmCC\" y=\"-7.041719\"/>\r\n   </g>\r\n   <g id=\"matplotlib.axis_1\">\r\n    <g id=\"xtick_1\">\r\n     <g id=\"line2d_1\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL 0 3.5 \r\n\" id=\"mb5cf5f59a6\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.3225\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_1\">\r\n      <!-- 0 -->\r\n      <defs>\r\n       <path d=\"M 31.78125 66.40625 \r\nQ 24.171875 66.40625 20.328125 58.90625 \r\nQ 16.5 51.421875 16.5 36.375 \r\nQ 16.5 21.390625 20.328125 13.890625 \r\nQ 24.171875 6.390625 31.78125 6.390625 \r\nQ 39.453125 6.390625 43.28125 13.890625 \r\nQ 47.125 21.390625 47.125 36.375 \r\nQ 47.125 51.421875 43.28125 58.90625 \r\nQ 39.453125 66.40625 31.78125 66.40625 \r\nz\r\nM 31.78125 74.21875 \r\nQ 44.046875 74.21875 50.515625 64.515625 \r\nQ 56.984375 54.828125 56.984375 36.375 \r\nQ 56.984375 17.96875 50.515625 8.265625 \r\nQ 44.046875 -1.421875 31.78125 -1.421875 \r\nQ 19.53125 -1.421875 13.0625 8.265625 \r\nQ 6.59375 17.96875 6.59375 36.375 \r\nQ 6.59375 54.828125 13.0625 64.515625 \r\nQ 19.53125 74.21875 31.78125 74.21875 \r\nz\r\n\" id=\"DejaVuSans-48\"/>\r\n      </defs>\r\n      <g transform=\"translate(27.14125 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_2\">\r\n     <g id=\"line2d_2\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"64.2975\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_2\">\r\n      <!-- 5 -->\r\n      <defs>\r\n       <path d=\"M 10.796875 72.90625 \r\nL 49.515625 72.90625 \r\nL 49.515625 64.59375 \r\nL 19.828125 64.59375 \r\nL 19.828125 46.734375 \r\nQ 21.96875 47.46875 24.109375 47.828125 \r\nQ 26.265625 48.1875 28.421875 48.1875 \r\nQ 40.625 48.1875 47.75 41.5 \r\nQ 54.890625 34.8125 54.890625 23.390625 \r\nQ 54.890625 11.625 47.5625 5.09375 \r\nQ 40.234375 -1.421875 26.90625 -1.421875 \r\nQ 22.3125 -1.421875 17.546875 -0.640625 \r\nQ 12.796875 0.140625 7.71875 1.703125 \r\nL 7.71875 11.625 \r\nQ 12.109375 9.234375 16.796875 8.0625 \r\nQ 21.484375 6.890625 26.703125 6.890625 \r\nQ 35.15625 6.890625 40.078125 11.328125 \r\nQ 45.015625 15.765625 45.015625 23.390625 \r\nQ 45.015625 31 40.078125 35.4375 \r\nQ 35.15625 39.890625 26.703125 39.890625 \r\nQ 22.75 39.890625 18.8125 39.015625 \r\nQ 14.890625 38.140625 10.796875 36.28125 \r\nz\r\n\" id=\"DejaVuSans-53\"/>\r\n      </defs>\r\n      <g transform=\"translate(61.11625 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_3\">\r\n     <g id=\"line2d_3\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"98.2725\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_3\">\r\n      <!-- 10 -->\r\n      <defs>\r\n       <path d=\"M 12.40625 8.296875 \r\nL 28.515625 8.296875 \r\nL 28.515625 63.921875 \r\nL 10.984375 60.40625 \r\nL 10.984375 69.390625 \r\nL 28.421875 72.90625 \r\nL 38.28125 72.90625 \r\nL 38.28125 8.296875 \r\nL 54.390625 8.296875 \r\nL 54.390625 0 \r\nL 12.40625 0 \r\nz\r\n\" id=\"DejaVuSans-49\"/>\r\n      </defs>\r\n      <g transform=\"translate(91.91 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_4\">\r\n     <g id=\"line2d_4\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"132.2475\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_4\">\r\n      <!-- 15 -->\r\n      <g transform=\"translate(125.885 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_5\">\r\n     <g id=\"line2d_5\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"166.2225\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_5\">\r\n      <!-- 20 -->\r\n      <defs>\r\n       <path d=\"M 19.1875 8.296875 \r\nL 53.609375 8.296875 \r\nL 53.609375 0 \r\nL 7.328125 0 \r\nL 7.328125 8.296875 \r\nQ 12.9375 14.109375 22.625 23.890625 \r\nQ 32.328125 33.6875 34.8125 36.53125 \r\nQ 39.546875 41.84375 41.421875 45.53125 \r\nQ 43.3125 49.21875 43.3125 52.78125 \r\nQ 43.3125 58.59375 39.234375 62.25 \r\nQ 35.15625 65.921875 28.609375 65.921875 \r\nQ 23.96875 65.921875 18.8125 64.3125 \r\nQ 13.671875 62.703125 7.8125 59.421875 \r\nL 7.8125 69.390625 \r\nQ 13.765625 71.78125 18.9375 73 \r\nQ 24.125 74.21875 28.421875 74.21875 \r\nQ 39.75 74.21875 46.484375 68.546875 \r\nQ 53.21875 62.890625 53.21875 53.421875 \r\nQ 53.21875 48.921875 51.53125 44.890625 \r\nQ 49.859375 40.875 45.40625 35.40625 \r\nQ 44.1875 33.984375 37.640625 27.21875 \r\nQ 31.109375 20.453125 19.1875 8.296875 \r\nz\r\n\" id=\"DejaVuSans-50\"/>\r\n      </defs>\r\n      <g transform=\"translate(159.86 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_6\">\r\n     <g id=\"line2d_6\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"200.1975\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_6\">\r\n      <!-- 25 -->\r\n      <g transform=\"translate(193.835 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_7\">\r\n     <g id=\"line2d_7\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"234.1725\" xlink:href=\"#mb5cf5f59a6\" y=\"225.041719\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_7\">\r\n      <!-- 30 -->\r\n      <defs>\r\n       <path d=\"M 40.578125 39.3125 \r\nQ 47.65625 37.796875 51.625 33 \r\nQ 55.609375 28.21875 55.609375 21.1875 \r\nQ 55.609375 10.40625 48.1875 4.484375 \r\nQ 40.765625 -1.421875 27.09375 -1.421875 \r\nQ 22.515625 -1.421875 17.65625 -0.515625 \r\nQ 12.796875 0.390625 7.625 2.203125 \r\nL 7.625 11.71875 \r\nQ 11.71875 9.328125 16.59375 8.109375 \r\nQ 21.484375 6.890625 26.8125 6.890625 \r\nQ 36.078125 6.890625 40.9375 10.546875 \r\nQ 45.796875 14.203125 45.796875 21.1875 \r\nQ 45.796875 27.640625 41.28125 31.265625 \r\nQ 36.765625 34.90625 28.71875 34.90625 \r\nL 20.21875 34.90625 \r\nL 20.21875 43.015625 \r\nL 29.109375 43.015625 \r\nQ 36.375 43.015625 40.234375 45.921875 \r\nQ 44.09375 48.828125 44.09375 54.296875 \r\nQ 44.09375 59.90625 40.109375 62.90625 \r\nQ 36.140625 65.921875 28.71875 65.921875 \r\nQ 24.65625 65.921875 20.015625 65.03125 \r\nQ 15.375 64.15625 9.8125 62.3125 \r\nL 9.8125 71.09375 \r\nQ 15.4375 72.65625 20.34375 73.4375 \r\nQ 25.25 74.21875 29.59375 74.21875 \r\nQ 40.828125 74.21875 47.359375 69.109375 \r\nQ 53.90625 64.015625 53.90625 55.328125 \r\nQ 53.90625 49.265625 50.4375 45.09375 \r\nQ 46.96875 40.921875 40.578125 39.3125 \r\nz\r\n\" id=\"DejaVuSans-51\"/>\r\n      </defs>\r\n      <g transform=\"translate(227.81 239.640156)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-51\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"matplotlib.axis_2\">\r\n    <g id=\"ytick_1\">\r\n     <g id=\"line2d_8\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL -3.5 0 \r\n\" id=\"m86e7862333\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"10.999219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_8\">\r\n      <!-- 0 -->\r\n      <g transform=\"translate(13.5625 14.798437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_2\">\r\n     <g id=\"line2d_9\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"44.974219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_9\">\r\n      <!-- 5 -->\r\n      <g transform=\"translate(13.5625 48.773437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_3\">\r\n     <g id=\"line2d_10\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"78.949219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_10\">\r\n      <!-- 10 -->\r\n      <g transform=\"translate(7.2 82.748437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_4\">\r\n     <g id=\"line2d_11\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"112.924219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_11\">\r\n      <!-- 15 -->\r\n      <g transform=\"translate(7.2 116.723437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_5\">\r\n     <g id=\"line2d_12\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"146.899219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_12\">\r\n      <!-- 20 -->\r\n      <g transform=\"translate(7.2 150.698437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_6\">\r\n     <g id=\"line2d_13\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"180.874219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_13\">\r\n      <!-- 25 -->\r\n      <g transform=\"translate(7.2 184.673437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_7\">\r\n     <g id=\"line2d_14\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"26.925\" xlink:href=\"#m86e7862333\" y=\"214.849219\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_14\">\r\n      <!-- 30 -->\r\n      <g transform=\"translate(7.2 218.648437)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-51\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"patch_3\">\r\n    <path d=\"M 26.925 225.041719 \r\nL 26.925 7.601719 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_4\">\r\n    <path d=\"M 244.365 225.041719 \r\nL 244.365 7.601719 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_5\">\r\n    <path d=\"M 26.925 225.041719 \r\nL 244.365 225.041719 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_6\">\r\n    <path d=\"M 26.925 7.601719 \r\nL 244.365 7.601719 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n  </g>\r\n </g>\r\n <defs>\r\n  <clipPath id=\"pf981f26bff\">\r\n   <rect height=\"217.44\" width=\"217.44\" x=\"26.925\" y=\"7.601719\"/>\r\n  </clipPath>\r\n </defs>\r\n</svg>\r\n",
      "text/plain": "<Figure size 432x288 with 1 Axes>"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(image.transpose((1, 2, 0)).asnumpy(), 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Defining data loaders__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "train_data = gluon.data.DataLoader(train_mnist, batch_size = batch_size, shuffle = True)\n",
    "test_data = gluon.data.DataLoader(test_mnist, batch_size = batch_size, shuffle = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Defining VGG Net Model Architecture__\n",
    "There are many versions of vgg architecture, below we are gonna workout with vgg 11, as it has 8 convolutional layers and 3 dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vggblock(layers, channels):\n",
    "    block = nn.HybridSequential()\n",
    "    for _ in range(layers):\n",
    "        block.add(nn.Conv2D(channels = channels, kernel_size = 3, padding = 1))\n",
    "        block.add(nn.BatchNorm())\n",
    "        # batchnorm is originally not the part of vgg net, but we are adding to each block as it speeds up the training\n",
    "        block.add(nn.Activation('relu'))\n",
    "    block.add(nn.MaxPool2D(pool_size = 2, strides = 2))\n",
    "    return block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "HybridSequential(\n  (0): HybridSequential(\n    (0): Conv2D(None -> 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (2): Activation(relu)\n    (3): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False, global_pool=False, pool_type=max, layout=NCHW)\n  )\n  (1): HybridSequential(\n    (0): Conv2D(None -> 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (2): Activation(relu)\n    (3): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False, global_pool=False, pool_type=max, layout=NCHW)\n  )\n  (2): HybridSequential(\n    (0): Conv2D(None -> 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (2): Activation(relu)\n    (3): Conv2D(None -> 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (5): Activation(relu)\n    (6): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False, global_pool=False, pool_type=max, layout=NCHW)\n  )\n  (3): HybridSequential(\n    (0): Conv2D(None -> 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (2): Activation(relu)\n    (3): Conv2D(None -> 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (5): Activation(relu)\n    (6): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False, global_pool=False, pool_type=max, layout=NCHW)\n  )\n  (4): HybridSequential(\n    (0): Conv2D(None -> 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (2): Activation(relu)\n    (3): Conv2D(None -> 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n    (4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False, use_global_stats=False, in_channels=None)\n    (5): Activation(relu)\n    (6): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False, global_pool=False, pool_type=max, layout=NCHW)\n  )\n  (5): Flatten\n  (6): Dense(None -> 4096, Activation(relu))\n  (7): Dropout(p = 0.5, axes=())\n  (8): Dense(None -> 4096, Activation(relu))\n  (9): Dropout(p = 0.5, axes=())\n  (10): Dense(None -> 10, linear)\n)"
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = nn.HybridSequential()\n",
    "\n",
    "model.add(vggblock(layers = 1, channels = 64))\n",
    "model.add(vggblock(layers = 1, channels = 128))\n",
    "model.add(vggblock(layers = 2, channels = 256))\n",
    "model.add(vggblock(layers = 2, channels = 512))\n",
    "model.add(vggblock(layers = 2, channels = 512))\n",
    "\n",
    "model.add(nn.Flatten())\n",
    "\n",
    "model.add(nn.Dense(4096, activation = 'relu'))\n",
    "model.add(nn.Dropout(0.5))\n",
    "model.add(nn.Dense(4096, activation = 'relu'))\n",
    "model.add(nn.Dropout(0.5))\n",
    "model.add(nn.Dense(10))\n",
    "\n",
    "model.initialize(mx.init.Xavier(), ctx)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Summarizing the model__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "--------------------------------------------------------------------------------\n        Layer (type)                                Output Shape         Param #\n================================================================================\n               Input                              (1, 3, 32, 32)               0\n            Conv2D-1                             (1, 64, 32, 32)            1792\n         BatchNorm-2                             (1, 64, 32, 32)             256\n        Activation-3                             (1, 64, 32, 32)               0\n         MaxPool2D-4                             (1, 64, 16, 16)               0\n            Conv2D-5                            (1, 128, 16, 16)           73856\n         BatchNorm-6                            (1, 128, 16, 16)             512\n        Activation-7                            (1, 128, 16, 16)               0\n         MaxPool2D-8                              (1, 128, 8, 8)               0\n            Conv2D-9                              (1, 256, 8, 8)          295168\n        BatchNorm-10                              (1, 256, 8, 8)            1024\n       Activation-11                              (1, 256, 8, 8)               0\n           Conv2D-12                              (1, 256, 8, 8)          590080\n        BatchNorm-13                              (1, 256, 8, 8)            1024\n       Activation-14                              (1, 256, 8, 8)               0\n        MaxPool2D-15                              (1, 256, 4, 4)               0\n           Conv2D-16                              (1, 512, 4, 4)         1180160\n        BatchNorm-17                              (1, 512, 4, 4)            2048\n       Activation-18                              (1, 512, 4, 4)               0\n           Conv2D-19                              (1, 512, 4, 4)         2359808\n        BatchNorm-20                              (1, 512, 4, 4)            2048\n       Activation-21                              (1, 512, 4, 4)               0\n        MaxPool2D-22                              (1, 512, 2, 2)               0\n           Conv2D-23                              (1, 512, 2, 2)         2359808\n        BatchNorm-24                              (1, 512, 2, 2)            2048\n       Activation-25                              (1, 512, 2, 2)               0\n           Conv2D-26                              (1, 512, 2, 2)         2359808\n        BatchNorm-27                              (1, 512, 2, 2)            2048\n       Activation-28                              (1, 512, 2, 2)               0\n        MaxPool2D-29                              (1, 512, 1, 1)               0\n          Flatten-30                                    (1, 512)               0\n       Activation-31                   <Symbol dense12_relu_fwd>               0\n       Activation-32                                   (1, 4096)               0\n            Dense-33                                   (1, 4096)         2101248\n          Dropout-34                                   (1, 4096)               0\n       Activation-35                   <Symbol dense13_relu_fwd>               0\n       Activation-36                                   (1, 4096)               0\n            Dense-37                                   (1, 4096)        16781312\n          Dropout-38                                   (1, 4096)               0\n            Dense-39                                     (1, 10)           40970\n================================================================================\nParameters in forward computation graph, duplicate included\n   Total params: 28155018\n   Trainable params: 28149514\n   Non-trainable params: 5504\nShared params in forward computation graph: 0\nUnique parameters in model: 28155018\n--------------------------------------------------------------------------------\n"
    }
   ],
   "source": [
    "model.summary(image.expand_dims(0).as_in_context(ctx))"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Visualizing model architecture__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n -->\r\n<!-- Title: plot Pages: 1 -->\r\n<svg width=\"102pt\" height=\"3544pt\"\r\n viewBox=\"0.00 0.00 102.00 3544.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 3540)\">\r\n<title>plot</title>\r\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-3540 98,-3540 98,4 -4,4\"/>\r\n<!-- data -->\r\n<g id=\"node1\" class=\"node\"><title>data</title>\r\n<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"47\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-25.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data</text>\r\n</g>\r\n<!-- conv32_fwd -->\r\n<g id=\"node2\" class=\"node\"><title>conv32_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-152 -7.10543e-015,-152 -7.10543e-015,-94 94,-94 94,-152\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 64</text>\r\n</g>\r\n<!-- conv32_fwd&#45;&gt;data -->\r\n<g id=\"edge1\" class=\"edge\"><title>conv32_fwd&#45;&gt;data</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-83.7443C47,-75.2043 47,-66.2977 47,-58.2479\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-93.8971 42.5001,-83.897 47,-88.8971 47.0001,-83.8971 47.0001,-83.8971 47.0001,-83.8971 47,-88.8971 51.5001,-83.8971 47,-93.8971 47,-93.8971\"/>\r\n</g>\r\n<!-- batchnorm0_fwd -->\r\n<g id=\"node3\" class=\"node\"><title>batchnorm0_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-246 -7.10543e-015,-246 -7.10543e-015,-188 94,-188 94,-246\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-213.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm0_fwd</text>\r\n</g>\r\n<!-- batchnorm0_fwd&#45;&gt;conv32_fwd -->\r\n<g id=\"edge2\" class=\"edge\"><title>batchnorm0_fwd&#45;&gt;conv32_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-177.744C47,-169.204 47,-160.298 47,-152.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-187.897 42.5001,-177.897 47,-182.897 47.0001,-177.897 47.0001,-177.897 47.0001,-177.897 47,-182.897 51.5001,-177.897 47,-187.897 47,-187.897\"/>\r\n</g>\r\n<!-- relu32_fwd -->\r\n<g id=\"node4\" class=\"node\"><title>relu32_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-340 -7.10543e-015,-340 -7.10543e-015,-282 94,-282 94,-340\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-314.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-299.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu32_fwd&#45;&gt;batchnorm0_fwd -->\r\n<g id=\"edge3\" class=\"edge\"><title>relu32_fwd&#45;&gt;batchnorm0_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-271.744C47,-263.204 47,-254.298 47,-246.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-281.897 42.5001,-271.897 47,-276.897 47.0001,-271.897 47.0001,-271.897 47.0001,-271.897 47,-276.897 51.5001,-271.897 47,-281.897 47,-281.897\"/>\r\n</g>\r\n<!-- pool20_fwd -->\r\n<g id=\"node5\" class=\"node\"><title>pool20_fwd</title>\r\n<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-434 -7.10543e-015,-434 -7.10543e-015,-376 94,-376 94,-434\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-408.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-393.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 2x2/2x2</text>\r\n</g>\r\n<!-- pool20_fwd&#45;&gt;relu32_fwd -->\r\n<g id=\"edge4\" class=\"edge\"><title>pool20_fwd&#45;&gt;relu32_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-365.744C47,-357.204 47,-348.298 47,-340.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-375.897 42.5001,-365.897 47,-370.897 47.0001,-365.897 47.0001,-365.897 47.0001,-365.897 47,-370.897 51.5001,-365.897 47,-375.897 47,-375.897\"/>\r\n</g>\r\n<!-- conv33_fwd -->\r\n<g id=\"node6\" class=\"node\"><title>conv33_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-528 -7.10543e-015,-528 -7.10543e-015,-470 94,-470 94,-528\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-502.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-487.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 128</text>\r\n</g>\r\n<!-- conv33_fwd&#45;&gt;pool20_fwd -->\r\n<g id=\"edge5\" class=\"edge\"><title>conv33_fwd&#45;&gt;pool20_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-459.744C47,-451.204 47,-442.298 47,-434.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-469.897 42.5001,-459.897 47,-464.897 47.0001,-459.897 47.0001,-459.897 47.0001,-459.897 47,-464.897 51.5001,-459.897 47,-469.897 47,-469.897\"/>\r\n</g>\r\n<!-- batchnorm1_fwd -->\r\n<g id=\"node7\" class=\"node\"><title>batchnorm1_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-622 -7.10543e-015,-622 -7.10543e-015,-564 94,-564 94,-622\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-589.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm1_fwd</text>\r\n</g>\r\n<!-- batchnorm1_fwd&#45;&gt;conv33_fwd -->\r\n<g id=\"edge6\" class=\"edge\"><title>batchnorm1_fwd&#45;&gt;conv33_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-553.744C47,-545.204 47,-536.298 47,-528.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-563.897 42.5001,-553.897 47,-558.897 47.0001,-553.897 47.0001,-553.897 47.0001,-553.897 47,-558.897 51.5001,-553.897 47,-563.897 47,-563.897\"/>\r\n</g>\r\n<!-- relu33_fwd -->\r\n<g id=\"node8\" class=\"node\"><title>relu33_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-716 -7.10543e-015,-716 -7.10543e-015,-658 94,-658 94,-716\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-690.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-675.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu33_fwd&#45;&gt;batchnorm1_fwd -->\r\n<g id=\"edge7\" class=\"edge\"><title>relu33_fwd&#45;&gt;batchnorm1_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-647.744C47,-639.204 47,-630.298 47,-622.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-657.897 42.5001,-647.897 47,-652.897 47.0001,-647.897 47.0001,-647.897 47.0001,-647.897 47,-652.897 51.5001,-647.897 47,-657.897 47,-657.897\"/>\r\n</g>\r\n<!-- pool21_fwd -->\r\n<g id=\"node9\" class=\"node\"><title>pool21_fwd</title>\r\n<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-810 -7.10543e-015,-810 -7.10543e-015,-752 94,-752 94,-810\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-784.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-769.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 2x2/2x2</text>\r\n</g>\r\n<!-- pool21_fwd&#45;&gt;relu33_fwd -->\r\n<g id=\"edge8\" class=\"edge\"><title>pool21_fwd&#45;&gt;relu33_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-741.744C47,-733.204 47,-724.298 47,-716.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-751.897 42.5001,-741.897 47,-746.897 47.0001,-741.897 47.0001,-741.897 47.0001,-741.897 47,-746.897 51.5001,-741.897 47,-751.897 47,-751.897\"/>\r\n</g>\r\n<!-- conv34_fwd -->\r\n<g id=\"node10\" class=\"node\"><title>conv34_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-904 -7.10543e-015,-904 -7.10543e-015,-846 94,-846 94,-904\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-878.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-863.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 256</text>\r\n</g>\r\n<!-- conv34_fwd&#45;&gt;pool21_fwd -->\r\n<g id=\"edge9\" class=\"edge\"><title>conv34_fwd&#45;&gt;pool21_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-835.744C47,-827.204 47,-818.298 47,-810.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-845.897 42.5001,-835.897 47,-840.897 47.0001,-835.897 47.0001,-835.897 47.0001,-835.897 47,-840.897 51.5001,-835.897 47,-845.897 47,-845.897\"/>\r\n</g>\r\n<!-- batchnorm2_fwd -->\r\n<g id=\"node11\" class=\"node\"><title>batchnorm2_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-998 -7.10543e-015,-998 -7.10543e-015,-940 94,-940 94,-998\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-965.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm2_fwd</text>\r\n</g>\r\n<!-- batchnorm2_fwd&#45;&gt;conv34_fwd -->\r\n<g id=\"edge10\" class=\"edge\"><title>batchnorm2_fwd&#45;&gt;conv34_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-929.744C47,-921.204 47,-912.298 47,-904.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-939.897 42.5001,-929.897 47,-934.897 47.0001,-929.897 47.0001,-929.897 47.0001,-929.897 47,-934.897 51.5001,-929.897 47,-939.897 47,-939.897\"/>\r\n</g>\r\n<!-- relu34_fwd -->\r\n<g id=\"node12\" class=\"node\"><title>relu34_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1092 -7.10543e-015,-1092 -7.10543e-015,-1034 94,-1034 94,-1092\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1066.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1051.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu34_fwd&#45;&gt;batchnorm2_fwd -->\r\n<g id=\"edge11\" class=\"edge\"><title>relu34_fwd&#45;&gt;batchnorm2_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1023.74C47,-1015.2 47,-1006.3 47,-998.248\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1033.9 42.5001,-1023.9 47,-1028.9 47.0001,-1023.9 47.0001,-1023.9 47.0001,-1023.9 47,-1028.9 51.5001,-1023.9 47,-1033.9 47,-1033.9\"/>\r\n</g>\r\n<!-- conv35_fwd -->\r\n<g id=\"node13\" class=\"node\"><title>conv35_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1186 -7.10543e-015,-1186 -7.10543e-015,-1128 94,-1128 94,-1186\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1160.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1145.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 256</text>\r\n</g>\r\n<!-- conv35_fwd&#45;&gt;relu34_fwd -->\r\n<g id=\"edge12\" class=\"edge\"><title>conv35_fwd&#45;&gt;relu34_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1117.74C47,-1109.2 47,-1100.3 47,-1092.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1127.9 42.5001,-1117.9 47,-1122.9 47.0001,-1117.9 47.0001,-1117.9 47.0001,-1117.9 47,-1122.9 51.5001,-1117.9 47,-1127.9 47,-1127.9\"/>\r\n</g>\r\n<!-- batchnorm3_fwd -->\r\n<g id=\"node14\" class=\"node\"><title>batchnorm3_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-1280 -7.10543e-015,-1280 -7.10543e-015,-1222 94,-1222 94,-1280\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1247.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm3_fwd</text>\r\n</g>\r\n<!-- batchnorm3_fwd&#45;&gt;conv35_fwd -->\r\n<g id=\"edge13\" class=\"edge\"><title>batchnorm3_fwd&#45;&gt;conv35_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1211.74C47,-1203.2 47,-1194.3 47,-1186.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1221.9 42.5001,-1211.9 47,-1216.9 47.0001,-1211.9 47.0001,-1211.9 47.0001,-1211.9 47,-1216.9 51.5001,-1211.9 47,-1221.9 47,-1221.9\"/>\r\n</g>\r\n<!-- relu35_fwd -->\r\n<g id=\"node15\" class=\"node\"><title>relu35_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1374 -7.10543e-015,-1374 -7.10543e-015,-1316 94,-1316 94,-1374\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1348.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1333.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu35_fwd&#45;&gt;batchnorm3_fwd -->\r\n<g id=\"edge14\" class=\"edge\"><title>relu35_fwd&#45;&gt;batchnorm3_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1305.74C47,-1297.2 47,-1288.3 47,-1280.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1315.9 42.5001,-1305.9 47,-1310.9 47.0001,-1305.9 47.0001,-1305.9 47.0001,-1305.9 47,-1310.9 51.5001,-1305.9 47,-1315.9 47,-1315.9\"/>\r\n</g>\r\n<!-- pool22_fwd -->\r\n<g id=\"node16\" class=\"node\"><title>pool22_fwd</title>\r\n<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-1468 -7.10543e-015,-1468 -7.10543e-015,-1410 94,-1410 94,-1468\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1442.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1427.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 2x2/2x2</text>\r\n</g>\r\n<!-- pool22_fwd&#45;&gt;relu35_fwd -->\r\n<g id=\"edge15\" class=\"edge\"><title>pool22_fwd&#45;&gt;relu35_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1399.74C47,-1391.2 47,-1382.3 47,-1374.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1409.9 42.5001,-1399.9 47,-1404.9 47.0001,-1399.9 47.0001,-1399.9 47.0001,-1399.9 47,-1404.9 51.5001,-1399.9 47,-1409.9 47,-1409.9\"/>\r\n</g>\r\n<!-- conv36_fwd -->\r\n<g id=\"node17\" class=\"node\"><title>conv36_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1562 -7.10543e-015,-1562 -7.10543e-015,-1504 94,-1504 94,-1562\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1536.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1521.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 512</text>\r\n</g>\r\n<!-- conv36_fwd&#45;&gt;pool22_fwd -->\r\n<g id=\"edge16\" class=\"edge\"><title>conv36_fwd&#45;&gt;pool22_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1493.74C47,-1485.2 47,-1476.3 47,-1468.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1503.9 42.5001,-1493.9 47,-1498.9 47.0001,-1493.9 47.0001,-1493.9 47.0001,-1493.9 47,-1498.9 51.5001,-1493.9 47,-1503.9 47,-1503.9\"/>\r\n</g>\r\n<!-- batchnorm4_fwd -->\r\n<g id=\"node18\" class=\"node\"><title>batchnorm4_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-1656 -7.10543e-015,-1656 -7.10543e-015,-1598 94,-1598 94,-1656\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1623.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm4_fwd</text>\r\n</g>\r\n<!-- batchnorm4_fwd&#45;&gt;conv36_fwd -->\r\n<g id=\"edge17\" class=\"edge\"><title>batchnorm4_fwd&#45;&gt;conv36_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1587.74C47,-1579.2 47,-1570.3 47,-1562.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1597.9 42.5001,-1587.9 47,-1592.9 47.0001,-1587.9 47.0001,-1587.9 47.0001,-1587.9 47,-1592.9 51.5001,-1587.9 47,-1597.9 47,-1597.9\"/>\r\n</g>\r\n<!-- relu36_fwd -->\r\n<g id=\"node19\" class=\"node\"><title>relu36_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1750 -7.10543e-015,-1750 -7.10543e-015,-1692 94,-1692 94,-1750\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1724.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1709.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu36_fwd&#45;&gt;batchnorm4_fwd -->\r\n<g id=\"edge18\" class=\"edge\"><title>relu36_fwd&#45;&gt;batchnorm4_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1681.74C47,-1673.2 47,-1664.3 47,-1656.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1691.9 42.5001,-1681.9 47,-1686.9 47.0001,-1681.9 47.0001,-1681.9 47.0001,-1681.9 47,-1686.9 51.5001,-1681.9 47,-1691.9 47,-1691.9\"/>\r\n</g>\r\n<!-- conv37_fwd -->\r\n<g id=\"node20\" class=\"node\"><title>conv37_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1844 -7.10543e-015,-1844 -7.10543e-015,-1786 94,-1786 94,-1844\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1818.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1803.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 512</text>\r\n</g>\r\n<!-- conv37_fwd&#45;&gt;relu36_fwd -->\r\n<g id=\"edge19\" class=\"edge\"><title>conv37_fwd&#45;&gt;relu36_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1775.74C47,-1767.2 47,-1758.3 47,-1750.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1785.9 42.5001,-1775.9 47,-1780.9 47.0001,-1775.9 47.0001,-1775.9 47.0001,-1775.9 47,-1780.9 51.5001,-1775.9 47,-1785.9 47,-1785.9\"/>\r\n</g>\r\n<!-- batchnorm5_fwd -->\r\n<g id=\"node21\" class=\"node\"><title>batchnorm5_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-1938 -7.10543e-015,-1938 -7.10543e-015,-1880 94,-1880 94,-1938\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1905.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm5_fwd</text>\r\n</g>\r\n<!-- batchnorm5_fwd&#45;&gt;conv37_fwd -->\r\n<g id=\"edge20\" class=\"edge\"><title>batchnorm5_fwd&#45;&gt;conv37_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1869.74C47,-1861.2 47,-1852.3 47,-1844.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1879.9 42.5001,-1869.9 47,-1874.9 47.0001,-1869.9 47.0001,-1869.9 47.0001,-1869.9 47,-1874.9 51.5001,-1869.9 47,-1879.9 47,-1879.9\"/>\r\n</g>\r\n<!-- relu37_fwd -->\r\n<g id=\"node22\" class=\"node\"><title>relu37_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-2032 -7.10543e-015,-2032 -7.10543e-015,-1974 94,-1974 94,-2032\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2006.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-1991.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu37_fwd&#45;&gt;batchnorm5_fwd -->\r\n<g id=\"edge21\" class=\"edge\"><title>relu37_fwd&#45;&gt;batchnorm5_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-1963.74C47,-1955.2 47,-1946.3 47,-1938.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-1973.9 42.5001,-1963.9 47,-1968.9 47.0001,-1963.9 47.0001,-1963.9 47.0001,-1963.9 47,-1968.9 51.5001,-1963.9 47,-1973.9 47,-1973.9\"/>\r\n</g>\r\n<!-- pool23_fwd -->\r\n<g id=\"node23\" class=\"node\"><title>pool23_fwd</title>\r\n<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-2126 -7.10543e-015,-2126 -7.10543e-015,-2068 94,-2068 94,-2126\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2100.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2085.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 2x2/2x2</text>\r\n</g>\r\n<!-- pool23_fwd&#45;&gt;relu37_fwd -->\r\n<g id=\"edge22\" class=\"edge\"><title>pool23_fwd&#45;&gt;relu37_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2057.74C47,-2049.2 47,-2040.3 47,-2032.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2067.9 42.5001,-2057.9 47,-2062.9 47.0001,-2057.9 47.0001,-2057.9 47.0001,-2057.9 47,-2062.9 51.5001,-2057.9 47,-2067.9 47,-2067.9\"/>\r\n</g>\r\n<!-- conv38_fwd -->\r\n<g id=\"node24\" class=\"node\"><title>conv38_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-2220 -7.10543e-015,-2220 -7.10543e-015,-2162 94,-2162 94,-2220\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2194.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2179.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 512</text>\r\n</g>\r\n<!-- conv38_fwd&#45;&gt;pool23_fwd -->\r\n<g id=\"edge23\" class=\"edge\"><title>conv38_fwd&#45;&gt;pool23_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2151.74C47,-2143.2 47,-2134.3 47,-2126.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2161.9 42.5001,-2151.9 47,-2156.9 47.0001,-2151.9 47.0001,-2151.9 47.0001,-2151.9 47,-2156.9 51.5001,-2151.9 47,-2161.9 47,-2161.9\"/>\r\n</g>\r\n<!-- batchnorm6_fwd -->\r\n<g id=\"node25\" class=\"node\"><title>batchnorm6_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-2314 -7.10543e-015,-2314 -7.10543e-015,-2256 94,-2256 94,-2314\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2281.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm6_fwd</text>\r\n</g>\r\n<!-- batchnorm6_fwd&#45;&gt;conv38_fwd -->\r\n<g id=\"edge24\" class=\"edge\"><title>batchnorm6_fwd&#45;&gt;conv38_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2245.74C47,-2237.2 47,-2228.3 47,-2220.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2255.9 42.5001,-2245.9 47,-2250.9 47.0001,-2245.9 47.0001,-2245.9 47.0001,-2245.9 47,-2250.9 51.5001,-2245.9 47,-2255.9 47,-2255.9\"/>\r\n</g>\r\n<!-- relu38_fwd -->\r\n<g id=\"node26\" class=\"node\"><title>relu38_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-2408 -7.10543e-015,-2408 -7.10543e-015,-2350 94,-2350 94,-2408\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2382.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2367.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu38_fwd&#45;&gt;batchnorm6_fwd -->\r\n<g id=\"edge25\" class=\"edge\"><title>relu38_fwd&#45;&gt;batchnorm6_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2339.74C47,-2331.2 47,-2322.3 47,-2314.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2349.9 42.5001,-2339.9 47,-2344.9 47.0001,-2339.9 47.0001,-2339.9 47.0001,-2339.9 47,-2344.9 51.5001,-2339.9 47,-2349.9 47,-2349.9\"/>\r\n</g>\r\n<!-- conv39_fwd -->\r\n<g id=\"node27\" class=\"node\"><title>conv39_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-2502 -7.10543e-015,-2502 -7.10543e-015,-2444 94,-2444 94,-2502\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2476.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Convolution</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2461.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">3x3/1x1, 512</text>\r\n</g>\r\n<!-- conv39_fwd&#45;&gt;relu38_fwd -->\r\n<g id=\"edge26\" class=\"edge\"><title>conv39_fwd&#45;&gt;relu38_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2433.74C47,-2425.2 47,-2416.3 47,-2408.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2443.9 42.5001,-2433.9 47,-2438.9 47.0001,-2433.9 47.0001,-2433.9 47.0001,-2433.9 47,-2438.9 51.5001,-2433.9 47,-2443.9 47,-2443.9\"/>\r\n</g>\r\n<!-- batchnorm7_fwd -->\r\n<g id=\"node28\" class=\"node\"><title>batchnorm7_fwd</title>\r\n<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-2596 -7.10543e-015,-2596 -7.10543e-015,-2538 94,-2538 94,-2596\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2563.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">batchnorm7_fwd</text>\r\n</g>\r\n<!-- batchnorm7_fwd&#45;&gt;conv39_fwd -->\r\n<g id=\"edge27\" class=\"edge\"><title>batchnorm7_fwd&#45;&gt;conv39_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2527.74C47,-2519.2 47,-2510.3 47,-2502.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2537.9 42.5001,-2527.9 47,-2532.9 47.0001,-2527.9 47.0001,-2527.9 47.0001,-2527.9 47,-2532.9 51.5001,-2527.9 47,-2537.9 47,-2537.9\"/>\r\n</g>\r\n<!-- relu39_fwd -->\r\n<g id=\"node29\" class=\"node\"><title>relu39_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-2690 -7.10543e-015,-2690 -7.10543e-015,-2632 94,-2632 94,-2690\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2664.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2649.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- relu39_fwd&#45;&gt;batchnorm7_fwd -->\r\n<g id=\"edge28\" class=\"edge\"><title>relu39_fwd&#45;&gt;batchnorm7_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2621.74C47,-2613.2 47,-2604.3 47,-2596.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2631.9 42.5001,-2621.9 47,-2626.9 47.0001,-2621.9 47.0001,-2621.9 47.0001,-2621.9 47,-2626.9 51.5001,-2621.9 47,-2631.9 47,-2631.9\"/>\r\n</g>\r\n<!-- pool24_fwd -->\r\n<g id=\"node30\" class=\"node\"><title>pool24_fwd</title>\r\n<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-2784 -7.10543e-015,-2784 -7.10543e-015,-2726 94,-2726 94,-2784\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2758.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Pooling</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2743.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">max, 2x2/2x2</text>\r\n</g>\r\n<!-- pool24_fwd&#45;&gt;relu39_fwd -->\r\n<g id=\"edge29\" class=\"edge\"><title>pool24_fwd&#45;&gt;relu39_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2715.74C47,-2707.2 47,-2698.3 47,-2690.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2725.9 42.5001,-2715.9 47,-2720.9 47.0001,-2715.9 47.0001,-2715.9 47.0001,-2715.9 47,-2720.9 51.5001,-2715.9 47,-2725.9 47,-2725.9\"/>\r\n</g>\r\n<!-- flatten4_flatten0 -->\r\n<g id=\"node31\" class=\"node\"><title>flatten4_flatten0</title>\r\n<polygon fill=\"#fdb462\" stroke=\"black\" points=\"94,-2878 -7.10543e-015,-2878 -7.10543e-015,-2820 94,-2820 94,-2878\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2845.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">flatten4_flatten0</text>\r\n</g>\r\n<!-- flatten4_flatten0&#45;&gt;pool24_fwd -->\r\n<g id=\"edge30\" class=\"edge\"><title>flatten4_flatten0&#45;&gt;pool24_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2809.74C47,-2801.2 47,-2792.3 47,-2784.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2819.9 42.5001,-2809.9 47,-2814.9 47.0001,-2809.9 47.0001,-2809.9 47.0001,-2809.9 47,-2814.9 51.5001,-2809.9 47,-2819.9 47,-2819.9\"/>\r\n</g>\r\n<!-- dense12_fwd -->\r\n<g id=\"node32\" class=\"node\"><title>dense12_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-2972 -7.10543e-015,-2972 -7.10543e-015,-2914 94,-2914 94,-2972\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2946.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-2931.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">4096</text>\r\n</g>\r\n<!-- dense12_fwd&#45;&gt;flatten4_flatten0 -->\r\n<g id=\"edge31\" class=\"edge\"><title>dense12_fwd&#45;&gt;flatten4_flatten0</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2903.74C47,-2895.2 47,-2886.3 47,-2878.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-2913.9 42.5001,-2903.9 47,-2908.9 47.0001,-2903.9 47.0001,-2903.9 47.0001,-2903.9 47,-2908.9 51.5001,-2903.9 47,-2913.9 47,-2913.9\"/>\r\n</g>\r\n<!-- dense12_relu_fwd -->\r\n<g id=\"node33\" class=\"node\"><title>dense12_relu_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-3066 -7.10543e-015,-3066 -7.10543e-015,-3008 94,-3008 94,-3066\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3040.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3025.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- dense12_relu_fwd&#45;&gt;dense12_fwd -->\r\n<g id=\"edge32\" class=\"edge\"><title>dense12_relu_fwd&#45;&gt;dense12_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-2997.74C47,-2989.2 47,-2980.3 47,-2972.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3007.9 42.5001,-2997.9 47,-3002.9 47.0001,-2997.9 47.0001,-2997.9 47.0001,-2997.9 47,-3002.9 51.5001,-2997.9 47,-3007.9 47,-3007.9\"/>\r\n</g>\r\n<!-- dropout8_fwd -->\r\n<g id=\"node34\" class=\"node\"><title>dropout8_fwd</title>\r\n<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-3160 -7.10543e-015,-3160 -7.10543e-015,-3102 94,-3102 94,-3160\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3127.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">dropout8_fwd</text>\r\n</g>\r\n<!-- dropout8_fwd&#45;&gt;dense12_relu_fwd -->\r\n<g id=\"edge33\" class=\"edge\"><title>dropout8_fwd&#45;&gt;dense12_relu_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-3091.74C47,-3083.2 47,-3074.3 47,-3066.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3101.9 42.5001,-3091.9 47,-3096.9 47.0001,-3091.9 47.0001,-3091.9 47.0001,-3091.9 47,-3096.9 51.5001,-3091.9 47,-3101.9 47,-3101.9\"/>\r\n</g>\r\n<!-- dense13_fwd -->\r\n<g id=\"node35\" class=\"node\"><title>dense13_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-3254 -7.10543e-015,-3254 -7.10543e-015,-3196 94,-3196 94,-3254\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3228.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3213.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">4096</text>\r\n</g>\r\n<!-- dense13_fwd&#45;&gt;dropout8_fwd -->\r\n<g id=\"edge34\" class=\"edge\"><title>dense13_fwd&#45;&gt;dropout8_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-3185.74C47,-3177.2 47,-3168.3 47,-3160.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3195.9 42.5001,-3185.9 47,-3190.9 47.0001,-3185.9 47.0001,-3185.9 47.0001,-3185.9 47,-3190.9 51.5001,-3185.9 47,-3195.9 47,-3195.9\"/>\r\n</g>\r\n<!-- dense13_relu_fwd -->\r\n<g id=\"node36\" class=\"node\"><title>dense13_relu_fwd</title>\r\n<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-3348 -7.10543e-015,-3348 -7.10543e-015,-3290 94,-3290 94,-3348\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3322.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">Activation</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3307.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">relu</text>\r\n</g>\r\n<!-- dense13_relu_fwd&#45;&gt;dense13_fwd -->\r\n<g id=\"edge35\" class=\"edge\"><title>dense13_relu_fwd&#45;&gt;dense13_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-3279.74C47,-3271.2 47,-3262.3 47,-3254.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3289.9 42.5001,-3279.9 47,-3284.9 47.0001,-3279.9 47.0001,-3279.9 47.0001,-3279.9 47,-3284.9 51.5001,-3279.9 47,-3289.9 47,-3289.9\"/>\r\n</g>\r\n<!-- dropout9_fwd -->\r\n<g id=\"node37\" class=\"node\"><title>dropout9_fwd</title>\r\n<polygon fill=\"#fccde5\" stroke=\"black\" points=\"94,-3442 -7.10543e-015,-3442 -7.10543e-015,-3384 94,-3384 94,-3442\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3409.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">dropout9_fwd</text>\r\n</g>\r\n<!-- dropout9_fwd&#45;&gt;dense13_relu_fwd -->\r\n<g id=\"edge36\" class=\"edge\"><title>dropout9_fwd&#45;&gt;dense13_relu_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-3373.74C47,-3365.2 47,-3356.3 47,-3348.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3383.9 42.5001,-3373.9 47,-3378.9 47.0001,-3373.9 47.0001,-3373.9 47.0001,-3373.9 47,-3378.9 51.5001,-3373.9 47,-3383.9 47,-3383.9\"/>\r\n</g>\r\n<!-- dense14_fwd -->\r\n<g id=\"node38\" class=\"node\"><title>dense14_fwd</title>\r\n<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-3536 -7.10543e-015,-3536 -7.10543e-015,-3478 94,-3478 94,-3536\"/>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3510.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">FullyConnected</text>\r\n<text text-anchor=\"middle\" x=\"47\" y=\"-3495.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">10</text>\r\n</g>\r\n<!-- dense14_fwd&#45;&gt;dropout9_fwd -->\r\n<g id=\"edge37\" class=\"edge\"><title>dense14_fwd&#45;&gt;dropout9_fwd</title>\r\n<path fill=\"none\" stroke=\"black\" d=\"M47,-3467.74C47,-3459.2 47,-3450.3 47,-3442.25\"/>\r\n<polygon fill=\"black\" stroke=\"black\" points=\"47,-3477.9 42.5001,-3467.9 47,-3472.9 47.0001,-3467.9 47.0001,-3467.9 47.0001,-3467.9 47,-3472.9 51.5001,-3467.9 47,-3477.9 47,-3477.9\"/>\r\n</g>\r\n</g>\r\n</svg>\r\n",
      "text/plain": "<graphviz.dot.Digraph at 0x18193c68fc8>"
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx.viz.plot_network(model(mx.sym.var(name = 'data')))"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "By hybridizing the model becomes almost 1.5x fast, as it converts dynamically defined model in to static graph. Its a really cool feature that MXNet provides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.hybridize()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Defining our objective loss and optimizer__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': 0.001})"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Training the model__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch: 1 | Loss: 2.03419 | Training Accuracy: 0.19000\nEpoch: 2 | Loss: 1.62698 | Training Accuracy: 0.36566\nEpoch: 3 | Loss: 1.26235 | Training Accuracy: 0.56012\nEpoch: 4 | Loss: 1.01815 | Training Accuracy: 0.65746\nEpoch: 5 | Loss: 0.83747 | Training Accuracy: 0.71998\nEpoch: 6 | Loss: 0.71464 | Training Accuracy: 0.76320\nEpoch: 7 | Loss: 0.64805 | Training Accuracy: 0.79212\nEpoch: 8 | Loss: 0.52588 | Training Accuracy: 0.83078\nEpoch: 9 | Loss: 0.54991 | Training Accuracy: 0.82610\nEpoch: 10 | Loss: 0.44426 | Training Accuracy: 0.86008\n"
    }
   ],
   "source": [
    "epochs = 10\n",
    "batches = len(train_data)\n",
    "acc = mx.metric.Accuracy()\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    acc.reset()\n",
    "    cum_loss = 0.0\n",
    "    for features, labels in train_data:\n",
    "        features = features.as_in_context(ctx)\n",
    "        labels = labels.as_in_context(ctx)\n",
    "\n",
    "        with autograd.record():\n",
    "            outputs = model(features)\n",
    "            loss = objective(outputs, labels)\n",
    "        loss.backward()\n",
    "        trainer.step(batch_size)\n",
    "\n",
    "        cum_loss += loss.mean()\n",
    "        acc.update(labels, outputs)\n",
    "    \n",
    "    print(f'Epoch: {epoch + 1} | Loss: {cum_loss.asscalar()/batches:.5f} | Training Accuracy: {acc.get()[1]:.5f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## __Testing the model__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "('accuracy', 0.7935)"
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc.reset()\n",
    "for features, labels in test_data:\n",
    "    features = features.as_in_context(ctx)\n",
    "    labels = labels.as_in_context(ctx)\n",
    "    outputs = model(features)\n",
    "    acc.update(labels, outputs)\n",
    "acc.get()"
   ]
  }
 ]
}