06. Making Deep Learing Learn Faster/02. Layer Normalization - MXNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Layer Normalization - MXNet (From Scratch).ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "PRwW2a0itwE-",
        "colab_type": "code",
        "outputId": "50f127dc-01dd-4bff-bc3d-b2be12990dea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        }
      },
      "source": [
        "!pip install mxnet-cu100"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: mxnet-cu100 in /usr/local/lib/python3.6/dist-packages (1.4.1)\n",
            "Requirement already satisfied: graphviz<0.9.0,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (0.8.4)\n",
            "Requirement already satisfied: requests>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (2.21.0)\n",
            "Requirement already satisfied: numpy<1.15.0,>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (1.14.6)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (2019.3.9)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (2.8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yD7crq8C2Gof",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import mxnet as mx\n",
        "from mxnet import nd, autograd, gluon\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "ctx = mx.gpu()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e9V_k_6s2xhX",
        "colab_type": "text"
      },
      "source": [
        "#Getting MNIST dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZLDyQYZo2wxw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def transform(data, label):\n",
        "    return data.reshape((784)).astype('float32')/255.0, label.astype('float32')\n",
        "\n",
        "train_mnist = gluon.data.vision.datasets.MNIST(train = True, transform = transform)\n",
        "test_mnist = gluon.data.vision.datasets.MNIST(train = False, transform = transform)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H81sRUEQuNTw",
        "colab_type": "code",
        "outputId": "0298a6bb-0c8e-45ac-dc7e-7d7629ff36ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "image, label = train_mnist[5]\n",
        "image.shape, label.shape"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((784,), ())"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWwYBdfFuerm",
        "colab_type": "code",
        "outputId": "0b87278f-9931-4d41-aefa-a812855c3642",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "plt.imshow(image.reshape((28, 28)).asnumpy(), cmap = 'gray')\n",
        "plt.title(label)\n",
        "plt.show()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD6RJREFUeJzt3X+sVGV+x/HPR8ANAlpZ6O2Na8Vd\n3CiFqIRoMaa1Ma6IWVEbdZEYStVrzRqqrUZim4g2xsV0bYwmJmxEceNqTUA0G+3iryxWzQZUVFR2\nQXMtInJj1Yo/Agt8+8c9172Ld565zJyZM/i8X8nkzpzvOXO+Gf1wzswzcx5HhADk56CqGwBQDcIP\nZIrwA5ki/ECmCD+QKcIPZIrwA5ki/Bmz/S3b99h+1/YO2+ttn5VY/xrbH9j+1PYy299qZ78oF+HP\n20hJWyT9taTDJP2rpIdtT9p3RdtnSlok6XRJR0n6rqSb2tUoyme+4YfBbL8m6aaIWLHP8l9I6o2I\nG4rHp0t6ICL+rII2UQKO/PiK7S5J35f0xhDlv5D06qDHr0rqsv3tdvSG8hF+SJJsj5L0gKTlEbFx\niFXGSvq/QY8H7o9rdW9oDcIP2T5I0s8l7ZJ0VY3VPpN06KDHA/d3tLA1tBDhz5xtS7pHUpekv42I\n39dY9Q1Jxw96fLyk7RHxvy1uES1C+HG3pOMk/TAivkysd7+kS21Psf0n6h8ZuK8N/aFF+LQ/Y7aP\nktQraaek3YNKV0h6TtKbkqZExP8U6/+TpOsljZa0QtI/RMTOdvaM8hB+IFOc9gOZIvxApgg/kCnC\nD2RqZDt3ZptPF4EWiwgPZ72mjvy2Z9n+re3Nthc181wA2qvhoT7bIyT9TtIZkt6TtFbS3Ih4M7EN\nR36gxdpx5D9J0uaIeCcidkl6SNKcJp4PQBs1E/4j1H8hiAHvFcv+iO0e2+tsr2tiXwBK1vIP/CJi\nqaSlEqf9QCdp5si/VdKRgx5/p1gG4ADQTPjXSjrG9tG2D5b0I0mPldMWgFZr+LQ/InbbvkrSrySN\nkLQsIoa6/BOADtTWX/Xxnh9ovbZ8yQfAgYvwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrw\nA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIP\nZIrwA5ki/ECmCD+QKcIPZKrhKboBSRo3blyyPnbs2Jq1s88+O7ntxIkTk/Xbb789Wd+5c2eynrum\nwm+7V9IOSXsk7Y6IGWU0BaD1yjjy/01EfFjC8wBoI97zA5lqNvwhabXtl2z3DLWC7R7b62yva3Jf\nAErU7Gn/qRGx1fafSnrS9saIWDN4hYhYKmmpJNmOJvcHoCRNHfkjYmvxt0/SI5JOKqMpAK3XcPht\nj7E9buC+pB9I2lBWYwBaq5nT/i5Jj9geeJ5fRMR/ldIV2mbSpEnJ+vXXX5+sz5w5M1mfOnXq/rY0\nbN3d3cn6woULW7bvb4KGwx8R70g6vsReALQRQ31Apgg/kCnCD2SK8AOZIvxAphzRvi/d8Q2/1jj2\n2GNr1q6++urktvPmzUvWR48enawXQ701bdmypWZtx44dyW2PO+64ZP3DD9O/JzvttNNq1jZu3Jjc\n9kAWEen/KAWO/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIpLd3eAww47LFlfsmRJsn7RRRfVrNW7\ntHazNm3alKyfeeaZNWujRo1KbltvLH7ChAlN1XPHkR/IFOEHMkX4gUwRfiBThB/IFOEHMkX4gUwx\nzt8BzjvvvGT9sssua1MnX/f2228n62eccUaynvo9/+TJkxvqCeXgyA9kivADmSL8QKYIP5Apwg9k\nivADmSL8QKYY5+8AF1xwQcueu7e3N1lfu3Ztsl5viu7UOH499a7Lj9aqe+S3vcx2n+0Ng5aNt/2k\n7U3F38Nb2yaAsg3ntP8+SbP2WbZI0tMRcYykp4vHAA4gdcMfEWskfbTP4jmSlhf3l0s6t+S+ALRY\no+/5uyJiW3H/A0ldtVa03SOpp8H9AGiRpj/wi4hITcAZEUslLZWYqBPoJI0O9W233S1Jxd++8loC\n0A6Nhv8xSfOL+/MlPVpOOwDape5pv+0HJZ0maYLt9yTdKOknkh62famkdyVd2Momv+kuv/zyZL2n\nJ/2RyerVq2vWNm/enNy2r6+6k7aurpofFaEN6oY/IubWKJ1eci8A2oiv9wKZIvxApgg/kCnCD2SK\n8AOZ4ie9HeD9999P1hcvXtyeRtps5syZVbeQNY78QKYIP5Apwg9kivADmSL8QKYIP5Apwg9kinH+\nzC1cuDBZHzNmTMv2PW3atKa2f+GFF5L1F198sann/6bjyA9kivADmSL8QKYIP5Apwg9kivADmSL8\nQKYY5z8AHHLIIcn6lClTatZuvPHG5LazZ89uqKcBBx2UPn7s3bu34eeud52DBQsWJOt79uxpeN85\n4MgPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmGOdvg1GjRiXrJ554YrK+YsWKZL27u7tm7csvv0xu\nW28svd5v4mfNmpWs1/uOQsrIken/Pc8///xk/Y477qhZ27VrV0M9fZPUPfLbXma7z/aGQcsW295q\ne31xa+6bIgDabjin/fdJGuqf9/+IiBOK2+PltgWg1eqGPyLWSPqoDb0AaKNmPvC7yvZrxduCw2ut\nZLvH9jrb65rYF4CSNRr+uyV9T9IJkrZJ+mmtFSNiaUTMiIgZDe4LQAs0FP6I2B4ReyJir6SfSTqp\n3LYAtFpD4bc9eGzpPEkbaq0LoDM5ItIr2A9KOk3SBEnbJd1YPD5BUkjqlXRFRGyruzM7vbMD1MEH\nH5ys1xsLX7lyZVP7v+mmm2rWnnnmmeS2zz//fLI+fvz4ZL3e80+dOjVZb6V58+bVrK1atSq57c6d\nO8tup20iwsNZr+6XfCJi7hCL79nvjgB0FL7eC2SK8AOZIvxApgg/kCnCD2Sq7lBfqTs7gIf6Uj/L\nvfnmm5PbXnfddU3t+4knnkjWL7nkkpq1Tz75JLntxIkTk/XHH0//Zmv69OnJeuqns7fddlty23rD\nhHPmzEnWU5566qlkfcmSJcn6xx9/3PC+JWn9+vVNbZ8y3KE+jvxApgg/kCnCD2SK8AOZIvxApgg/\nkCnCD2SKcf7CiBEjkvVbbrmlZu3aa69Nbvv5558n64sWLUrWH3rooWQ9NeY8Y0b6Akp33XVXsl5v\n+82bNyfrV155Zc3as88+m9z20EMPTdZPOeWUZD31k95zzjknue2YMWOS9Xq2bNmSrB999NFNPX8K\n4/wAkgg/kCnCD2SK8AOZIvxApgg/kCnCD2SKcf5Cajxaku68886atS+++CK5bU9PT7K+evXqZP3k\nk09O1hcsWFCzdtZZZyW3HT16dLJe71oF9957b7Jeb7y7KnPnDnVR6j+4+OKLm3r+a665Jlmv9/2I\nZjDODyCJ8AOZIvxApgg/kCnCD2SK8AOZIvxApoYzRfeRku6X1KX+KbmXRsQdtsdL+k9Jk9Q/TfeF\nEZG8mHknj/Nv25aeYTx1fft60zlv3LgxWa/32/HJkycn681YvHhxsn7rrbcm63v27CmxG5ShzHH+\n3ZL+OSKmSPpLST+2PUXSIklPR8Qxkp4uHgM4QNQNf0Rsi4iXi/s7JL0l6QhJcyQtL1ZbLuncVjUJ\noHz79Z7f9iRJJ0r6jaSuiBg4V/5A/W8LABwgRg53RdtjJa2QdHVEfGr/4W1FRESt9/O2eySlv9wO\noO2GdeS3PUr9wX8gIlYWi7fb7i7q3ZL6hto2IpZGxIyISF8JEkBb1Q2/+w/x90h6KyJuH1R6TNL8\n4v58SY+W3x6AVhnOUN+pkp6T9LqkvcXiG9T/vv9hSX8u6V31D/V9VOe5Onao75VXXknWp02b1qZO\nvq7eNNlr1qypWVu1alVy297e3mR99+7dyTo6z3CH+uq+54+I/5ZU68lO35+mAHQOvuEHZIrwA5ki\n/ECmCD+QKcIPZIrwA5ni0t2FcePGJevnnlv7d0vTp09PbtvXN+SXH7+ybNmyZD01Bbck7dq1K1lH\nXrh0N4Akwg9kivADmSL8QKYIP5Apwg9kivADmWKcH/iGYZwfQBLhBzJF+IFMEX4gU4QfyBThBzJF\n+IFMEX4gU4QfyBThBzJF+IFMEX4gU4QfyBThBzJF+IFM1Q2/7SNtP2v7Tdtv2P7HYvli21ttry9u\ns1vfLoCy1L2Yh+1uSd0R8bLtcZJeknSupAslfRYR/z7snXExD6Dlhnsxj5HDeKJtkrYV93fYfkvS\nEc21B6Bq+/We3/YkSSdK+k2x6Crbr9leZvvwGtv02F5ne11TnQIo1bCv4Wd7rKRfS7olIlba7pL0\noaSQ9G/qf2vw93Weg9N+oMWGe9o/rPDbHiXpl5J+FRG3D1GfJOmXETG1zvMQfqDFSruAp21LukfS\nW4ODX3wQOOA8SRv2t0kA1RnOp/2nSnpO0uuS9haLb5A0V9IJ6j/t75V0RfHhYOq5OPIDLVbqaX9Z\nCD/Qely3H0AS4QcyRfiBTBF+IFOEH8gU4QcyRfiBTBF+IFOEH8gU4QcyRfiBTBF+IFOEH8gU4Qcy\nVfcCniX7UNK7gx5PKJZ1ok7trVP7kuitUWX2dtRwV2zr7/m/tnN7XUTMqKyBhE7trVP7kuitUVX1\nxmk/kCnCD2Sq6vAvrXj/KZ3aW6f2JdFboyrprdL3/ACqU/WRH0BFCD+QqUrCb3uW7d/a3mx7URU9\n1GK71/brxbTjlc4vWMyB2Gd7w6Bl420/aXtT8XfIORIr6q0jpm1PTCtf6WvXadPdt/09v+0Rkn4n\n6QxJ70laK2luRLzZ1kZqsN0raUZEVP6FENt/JekzSfcPTIVm+zZJH0XET4p/OA+PiOs7pLfF2s9p\n21vUW61p5f9OFb52ZU53X4YqjvwnSdocEe9ExC5JD0maU0EfHS8i1kj6aJ/FcyQtL+4vV///PG1X\no7eOEBHbIuLl4v4OSQPTylf62iX6qkQV4T9C0pZBj99ThS/AEELSatsv2e6pupkhdA2aFu0DSV1V\nNjOEutO2t9M+08p3zGvXyHT3ZeMDv687NSKmSzpL0o+L09uOFP3v2TpprPZuSd9T/xyO2yT9tMpm\nimnlV0i6OiI+HVyr8rUboq9KXrcqwr9V0pGDHn+nWNYRImJr8bdP0iPqf5vSSbYPzJBc/O2ruJ+v\nRMT2iNgTEXsl/UwVvnbFtPIrJD0QESuLxZW/dkP1VdXrVkX410o6xvbRtg+W9CNJj1XQx9fYHlN8\nECPbYyT9QJ039fhjkuYX9+dLerTCXv5Ip0zbXmtaeVX82nXcdPcR0fabpNnq/8T/bUn/UkUPNfr6\nrqRXi9sbVfcm6UH1nwb+Xv2fjVwq6duSnpa0SdJTksZ3UG8/V/9U7q+pP2jdFfV2qvpP6V+TtL64\nza76tUv0Vcnrxtd7gUzxgR+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5n6f1r83bQR3u6bAAAAAElF\nTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_eoQ8Cx2w0x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64\n",
        "train_data = gluon.data.DataLoader(train_mnist, batch_size, True)\n",
        "test_data = gluon.data.DataLoader(test_mnist, batch_size, False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FqqqOgfG0P7f",
        "colab_type": "text"
      },
      "source": [
        "##Initializing model parameters and batch normal parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8bduuD24DOT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weight_scale = .01\n",
        "\n",
        "#  Allocate parameters for the first hidden layer\n",
        "W1 = nd.random_normal(shape = (784, 256), scale = weight_scale, ctx = ctx)\n",
        "b1 = nd.random_normal(shape = 256, scale = weight_scale, ctx = ctx)\n",
        "gamma1 = nd.random_normal(shape = 256, scale = weight_scale, ctx = ctx)\n",
        "beta1 = nd.random_normal(shape = 256, scale = weight_scale, ctx = ctx)\n",
        "\n",
        "\n",
        "#  Allocate parameters for the second hidden layer\n",
        "W2 = nd.random_normal(shape = (256, 128), scale = weight_scale, ctx = ctx)\n",
        "b2 = nd.random_normal(shape = 128, scale = weight_scale, ctx = ctx)\n",
        "gamma2 = nd.random_normal(shape = 128, scale = weight_scale, ctx = ctx)\n",
        "beta2 = nd.random_normal(shape = 128, scale = weight_scale, ctx = ctx)\n",
        "\n",
        "#  Allocate parameters for the output layer\n",
        "W3 = nd.random_normal(shape=(128, 10), scale = weight_scale, ctx = ctx)\n",
        "b3 = nd.random_normal(shape=10, scale = weight_scale, ctx = ctx)\n",
        "\n",
        "params = [W1, b1, gamma1, beta1, W2, b2, gamma2, beta2, W3, b3]\n",
        "\n",
        "for param in params:\n",
        "    param.attach_grad()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uc2JVOAX4zsR",
        "colab_type": "text"
      },
      "source": [
        "##Defining Layer Norm Layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "60aSuiux4DR9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def layernorm(layer, gamma, beta):\n",
        "    mean = nd.mean(layer, axis = 1, keepdims = True)\n",
        "    variance = nd.mean((layer - mean) ** 2, axis=1, keepdims = True)\n",
        "    layer_hat = (layer - mean) * 1.0 / nd.sqrt(variance + 1e-8)\n",
        "    out = gamma * layer_hat + beta\n",
        "    \n",
        "    return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uB9EJyuu4DM0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def relu(x):\n",
        "    return nd.maximum(x, 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwO0Qwmy5fyn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def softmax(linear):\n",
        "    exp = nd.exp(linear - nd.max(linear))\n",
        "    partition = nd.sum(exp).reshape((-1, 1))\n",
        "    return exp / partition"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yvQc2ogE0WeL",
        "colab_type": "text"
      },
      "source": [
        "##Building the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FdYyCb65fvS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def model(inputs):\n",
        "    layer1 = layernorm(nd.relu(nd.dot(inputs, W1) + b1), gamma1, beta1)\n",
        "    layer2 = layernorm(nd.relu(nd.dot(layer1, W2) + b2), gamma2, beta2)\n",
        "    return nd.softmax(nd.dot(layer2, W3) + b3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQtVGnWv8zcN",
        "colab_type": "code",
        "outputId": "0223e81f-73df-4249-ee47-39f7efeb6ee7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "for data, _ in train_data:\n",
        "    data = data.as_in_context(ctx)\n",
        "    break\n",
        "    \n",
        "print(data.shape)\n",
        "output = model(data)\n",
        "print(output.shape)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 784)\n",
            "(64, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjbezA0S5fwQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def categoricalcrossentropy(pred, label):\n",
        "    return -nd.sum(label * nd.log(pred))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzdO_SXx5fuG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sgd(params, lr):\n",
        "    for param in params:\n",
        "        param[:] = param - lr * param.grad"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ff2oB_yFWR9O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy(predictions, labels):\n",
        "    return nd.mean(predictions.argmax(1) == labels.argmax(1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eGXnJ59P0bKz",
        "colab_type": "text"
      },
      "source": [
        "##Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toHHcCNo6AMC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        },
        "outputId": "c8a1f941-9cb5-49d6-c1fe-2a8da3ae162c"
      },
      "source": [
        "epochs = 10\n",
        "num_batches = 60000/batch_size\n",
        "learning_rate = .001\n",
        "losses = []\n",
        "accs = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    cumulative_loss = 0\n",
        "    cumulative_acc = 0\n",
        "    for features, labels in train_data:\n",
        "        features = features.as_in_context(ctx)\n",
        "        labels = labels.as_in_context(ctx).one_hot(10)\n",
        "        with autograd.record():\n",
        "            output = model(features)\n",
        "            loss = categoricalcrossentropy(output, labels)\n",
        "        loss.backward()\n",
        "        sgd(params, learning_rate)\n",
        "        cumulative_loss += loss\n",
        "        cumulative_acc += accuracy(output, labels)\n",
        "    loss = cumulative_loss.asscalar()/num_batches\n",
        "    cc = cumulative_acc.asscalar()/num_batches\n",
        "    print(f'Epoch: {epoch} | Loss: {loss:.5f} | Accuracy: {acc:.5f}')\n",
        "    losses.append(loss)\n",
        "    accs.append(acc)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0 | Loss: 67.16122 | Accuracy: 0.63060\n",
            "Epoch: 1 | Loss: 15.44060 | Accuracy: 0.93407\n",
            "Epoch: 2 | Loss: 11.29232 | Accuracy: 0.95153\n",
            "Epoch: 3 | Loss: 9.13283 | Accuracy: 0.95967\n",
            "Epoch: 4 | Loss: 7.58525 | Accuracy: 0.96690\n",
            "Epoch: 5 | Loss: 6.48895 | Accuracy: 0.97102\n",
            "Epoch: 6 | Loss: 5.76430 | Accuracy: 0.97397\n",
            "Epoch: 7 | Loss: 5.11823 | Accuracy: 0.97662\n",
            "Epoch: 8 | Loss: 4.56834 | Accuracy: 0.97877\n",
            "Epoch: 9 | Loss: 4.12497 | Accuracy: 0.98115\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJcq8cBl6AP_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        },
        "outputId": "f07cd1d8-0998-447c-919a-bf50dba87b91"
      },
      "source": [
        "plt.plot(losses, c = 'r')\n",
        "plt.title('Training Loss')\n",
        "plt.show()\n",
        "plt.plot(accs, c = 'g')\n",
        "plt.title('Training Accuracy')\n",
        "plt.show()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGrxJREFUeJzt3XmQnPV95/H3R6PRNZKQhAahAzEY\nCXHrYHpiLykvy5E1MZerMHG8uOQUW3ISNosdF4Fs1pvaXSdF1lsmZMuFTRk7SowNNiErgm02LODs\nxl5jRkIIrJEsLCQkWceAbmEkjfTdP56nPT2jOXqOnqf76c+r6qmnn6efnv5Ogz79zPf3HIoIzMys\n9o3LugAzMxsdDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7rVJEkNko5KWjia25rVMvk4dBsL\nko6WLE4BjgOn0uVPRcRjY1/VyEn6PLAgIj6ZdS1m47MuwOpDREwtPpa0Dfi3EfG/+9te0viI6BqL\n2szywi0XqwqSPi/pCUnfknQEuFPSByT9WNJBSbsl/ZWkxnT78ZJCUku6/I30+e9LOiLp/0m6YKjb\nps/fKOlnkg5J+h+Sfijpk8P4nS6T9E9p/a9J+nDJczdJ6kjff6ekz6Trz5H0vfQ1+yX9n+F+plZ/\nHOhWTT4CfBM4C3gC6ALuAWYDVwMfAj41wOs/DnwOmAW8BfzXoW4r6Rzg28C96fu+CbQN9ReRNAF4\nBvgu0Ax8BnhC0qJ0k68Dd0XENOBK4J/S9fcCW9PXnAv8x6G+t9UvB7pVk3+OiH+IiNMR8cuIeDki\nXoqIrojYCjwC/MsBXv9kRLRHxEngMWDZMLa9CVgfEWvS5x4E3h7G73I1MAH4QkScTNtL3wc+lj5/\nErhU0rSI2B8R60rWzwMWRsSJiPAeupXNgW7VZEfpgqSLJX1X0h5Jh4H/QrLX3J89JY/fBab2t+EA\n284rrSOSowZ2llF7b/OAt6LnUQfbgfnp448AtwBvSfqBpF9L1z+Qbve8pJ9LuncY7211yoFu1aT3\nIVdfAV4HFkXEdOA/AapwDbuBBcUFSaI7hIfiF8B56euLFgK7ANK/PG4BziFpzTyerj8cEZ+JiBbg\nNuA+SQP9VWL2Kw50q2bTgEPAMUmXMHD/fLQ8A6yQdLOk8SQ9/OZBXtMgaVLJNBH4EckYwGclNUq6\nFvhNkj76ZEkflzQ9bescAU4DpO97YfpFcIjk0M7TlflVLW8c6FbNPgusJAm8r5AMlFZUROwFfgv4\nIvAOcCHwCslx8/25E/hlybQ5Io4DNwO3kvTg/wr4eERsSV+zEtietpLuSn8GwBLgBeAo8EPgoYj4\nv6P2C1qu+cQiswFIaiBpn9zuYLVq5z10s14kfUjSjLR18jmSI09+knFZZoNyoJud6ddJjgXvBP41\n8JG0hWJW1QZtuUhaQs/e5ftIjjb4m3R9C7ANuCMiDlSkSjMzG9SQeuhpP3EX8GvA3cD+iHhA0v3A\nzIi4rzJlmpnZYIYa6L8B/GlEXC1pM3BNROyWNBf4QUQsGej1s2fPjpaWlhEVbGZWb9auXft2RAx2\n+OyQr7b4MeBb6eM5EbE7fbwHmNPXCyStAlYBLFy4kPb29iG+pZlZfZO0vZztyh4UTS82dAvwnd7P\npac397mrHxGPRERrRLQ2Nw/6BWNmZsM0lKNcbgTWpSdeAOxNWy2k832jXZyZmZVvKIH+23S3WwCe\nJjnbjXS+ZrSKMjOzoSsr0CU1ATcAT5WsfgC4QdIW4Pp02czMMlLWoGhEHAPO7rXuHeC6ShRlZmZD\n5zNFzcxywoFuZpYTtRHo3/wmfPnLWVdhZlbVaiPQn3oKvvCFrKswM6tqtRHohQJs3QrvvJN1JWZm\nVas2Ar2tLZm//HK2dZiZVbHaCPSrrgLJgW5mNoDaCPTp02HJEge6mdkAaiPQIWm7/OQn4Hugmpn1\nqXYCvVCAvXth586sKzEzq0q1FeiQ7KWbmdkZaifQly6Fxkb30c3M+lE7gT5pElx5pQPdzKwftRPo\nkAyMtrfD6dNZV2JmVnVqK9ALBTh8GH72s6wrMTOrOrUX6OC2i5lZH2or0C+5BJqafKSLmVkfaivQ\nGxqSywB4D93M7Ay1FeiQDIy+8gqcOJF1JWZmVaX2Ar1QSML8tdeyrsTMrKrUZqCD2y5mZr3UXqC3\ntMDs2R4YNTPrpfYCXUr20r2HbmbWQ1mBLmmGpCclbZLUIekDkmZJek7SlnQ+s9LF/kqhABs3wtGj\nY/aWZmbVrtw99IeAZyPiYmAp0AHcDzwfEYuB59PlsdHWlpz+v27dmL2lmVm1GzTQJZ0FfBB4FCAi\nTkTEQeBWYHW62WrgtkoVeQYPjJqZnaGcPfQLgE7g65JekfRVSU3AnIjYnW6zB5jT14slrZLULqm9\ns7NzdKo+5xw4/3wPjJqZlSgn0McDK4CHI2I5cIxe7ZWICKDPe8NFxCMR0RoRrc3NzSOtt5sHRs3M\neign0HcCOyPipXT5SZKA3ytpLkA631eZEvtRKMCbb8Lbb4/p25qZVatBAz0i9gA7JC1JV10HbASe\nBlam61YCaypSYX/a2pK599LNzICknVKOPwAekzQB2Ar8DsmXwbcl3QVsB+6oTIn9uOqq5Jj0l1+G\nG28c07c2M6tGZQV6RKwHWvt46rrRLWcIpk2Diy/2HrqZWar2zhQt1daWHOkSfY7HmpnVldoO9EIB\n9u2DHTuyrsTMLHO1HejFgVEfj25mVuOBfuWV0NjoPrqZGbUe6BMnwtKlDnQzM2o90CFpu7S3Jxfr\nMjOrY7Uf6IUCHDkCmzdnXYmZWaZqP9B9xqiZGZCHQF+yBKZO9ZEuZlb3aj/QGxqSywB4D93M6lzt\nBzokbZf16+HEiawrMTPLTD4CvVBIwnzDhqwrMTPLTH4CHdx2MbO6lo9AP/98aG72wKiZ1bV8BLrk\nW9KZWd3LR6BDMjC6cWNykpGZWR3KT6AXCsl10dety7oSM7NM5CvQwW0XM6tb+Qn05mZoafHAqJnV\nrfwEOnhg1MzqWr4Cva0Ntm2Dzs6sKzEzG3P5CnT30c2sjpUV6JK2SXpN0npJ7em6WZKek7Qlnc+s\nbKllWLEiOSbdgW5mdWgoe+j/KiKWRURrunw/8HxELAaeT5ezNW0aXHqpB0bNrC6NpOVyK7A6fbwa\nuG3k5YyC4sBoRNaVmJmNqXIDPYB/lLRW0qp03ZyI2J0+3gPM6euFklZJapfU3jkWg5WFQjIo+tZb\nlX8vM7MqUm6g/3pErABuBO6W9MHSJyMiSEL/DBHxSES0RkRrc3PzyKotR/GWdG67mFmdKSvQI2JX\nOt8H/D3QBuyVNBcgne+rVJFDcuWVMGGCB0bNrO4MGuiSmiRNKz4GfgN4HXgaWJluthJYU6kih2TC\nBFi2zIFuZnVnfBnbzAH+XlJx+29GxLOSXga+LekuYDtwR+XKHKJCAVavhlOnknuOmpnVgUEDPSK2\nAkv7WP8OcF0lihqxQgG+9CXYvDk5jNHMrA7k60zRIg+MmlkdymegL1mSnGTkPrqZ1ZF8Bvq4cXDV\nVQ50M6sr+Qx0SNou69fD8eNZV2JmNibyG+iFApw8CRs2ZF2JmdmYyG+gFwdG3XYxszqR30A/7zw4\n5xwf6WJmdSO/gS75lnRmVlfyG+iQtF06OuDIkawrMTOruHwHeqGQXBd97dqsKzEzq7j8Bzq47WJm\ndSHfgT57NlxwgQdGzawu5DvQwQOjZlY38h/obW2wfTvsq477b5iZVUr+A919dDOrE/kP9BUrkot1\nOdDNLOfyH+hTpyY3ufDAqJnlXP4DHboHRiOyrsTMrGLqI9Db2uDtt5PBUTOznKqPQC8OjLrtYmY5\nVh+BfsUVMGGCB0bNLNfqI9AnTIDlyx3oZpZrZQe6pAZJr0h6Jl2+QNJLkt6Q9ISkCZUrcxQUCtDe\nDqdOZV2JmVlFDGUP/R6go2T5L4AHI2IRcAC4azQLG3WFAhw7Bps2ZV2JmVlFlBXokhYAHwa+mi4L\nuBZ4Mt1kNXBbJQocNcVb0nlg1Mxyqtw99L8E/gg4nS6fDRyMiK50eScwv68XSlolqV1Se2dn54iK\nHZGLLoLp091HN7PcGjTQJd0E7IuIYd0lIiIeiYjWiGhtbm4ezo8YHePGQWurA93McqucPfSrgVsk\nbQMeJ2m1PATMkDQ+3WYBsKsiFY6mQgFefRWOH8+6EjOzUTdooEfEH0fEgohoAT4GvBAR/wZ4Ebg9\n3WwlsKZiVY6WQgFOnkxC3cwsZ0ZyHPp9wB9KeoOkp/7o6JRUQcWBUbddzCyHhhToEfGDiLgpfbw1\nItoiYlFEfDQiqr+PsWABzJnjI13MLJfq40zRIinZS/ceupnlUH0FOiR99E2b4PDhrCsxMxtV9Rno\nEbB2WEdhmplVrfoMdHDbxcxyp/4C/eyz4X3v88ComeVO/QU6dN+SzswsR+oz0Nva4K23YO/erCsx\nMxs19Rno7qObWQ7VZ6CvWJFcrMuBbmY5Up+B3tQEl13mgVEzy5X6DHToHhiNyLoSM7NRUb+B3tYG\n77wD27ZlXYmZ2aio30AvDoy67WJmOVG/gX7FFTBxogdGzSw36jfQGxth+XIHupnlRv0GOiRtl7Vr\n4dSprCsxMxux+g70tjY4dgw6OrKuxMxsxOo70D0wamY5Ut+BvngxTJ/uPrqZ5UJ9B/q4cb7yopnl\nRn0HOiSB/uqr8N57WVdiZjYiDvS2NujqSkLdzKyGOdB9KV0zy4lBA13SJEk/kfSqpJ9K+s/p+gsk\nvSTpDUlPSJpQ+XIrYP58OPdcH+liZjWvnD3048C1EbEUWAZ8SNL7gb8AHoyIRcAB4K7KlVlBUtJ2\n8R66mdW4QQM9EkfTxcZ0CuBa4Ml0/WrgtopUOBYKBdi0CQ4dyroSM7NhK6uHLqlB0npgH/Ac8HPg\nYER0pZvsBOb389pVktoltXd2do5GzaOv2EdfuzbbOszMRqCsQI+IUxGxDFgAtAEXl/sGEfFIRLRG\nRGtzc/Mwy6wwD4yaWQ4M6SiXiDgIvAh8AJghaXz61AJg1yjXNnZmzYILL/TAqJnVtHKOcmmWNCN9\nPBm4AeggCfbb081WAmsqVeSY8MComdW4cvbQ5wIvStoAvAw8FxHPAPcBfyjpDeBs4NHKlTkGCgXY\nsQP27Mm6EjOzYRk/2AYRsQFY3sf6rST99Hwo7aPffHO2tZiZDYPPFC1avhwaGtx2MbOa5UAvamqC\nyy7zwKiZ1SwHeqnipXQjsq7EzGzIHOil2tpg/354882sKzEzGzIHeinfks7MapgDvdTll8OkSR4Y\nNbOa5EAv1diYHO3iPXQzq0EO9N4KBVi3LrmLkZlZDXGg99bWBu++Cx0dWVdiZjYkDvTePDBqZjXK\ngd7bokUwY4YHRs2s5jjQexs3DlpbHehmVnMc6H0pFGDDBnjvvawrMTMrmwO9L21tyVEu69dnXYmZ\nWdkc6H3xwKiZ1SAHel/mz4e5c91HN7Oa4kDvj29JZ2Y1xoHen0IBNm+GgwezrsTMrCwO9P60pXfX\nW7s22zrMzMrkQO9Pa2syd9vFzGqEA70/M2cmZ436SBczqxEO9IF4YNTMasiggS7pPEkvStoo6aeS\n7knXz5L0nKQt6Xxm5csdY4UC7NwJu3dnXYmZ2aDK2UPvAj4bEZcC7wfulnQpcD/wfEQsBp5Pl/Ol\nODDqvXQzqwGDBnpE7I6IdenjI0AHMB+4FVidbrYauK1SRWZm2TJoaHCgm1lNGFIPXVILsBx4CZgT\nEcVexB5gTj+vWSWpXVJ7Z2fnCErNwJQpyX1GPTBqZjWg7ECXNBX4O+DTEXG49LmICCD6el1EPBIR\nrRHR2tzcPKJiM9HWBu3tEH3+emZmVaOsQJfUSBLmj0XEU+nqvZLmps/PBfZVpsSMFQqwfz9s3Zp1\nJWZmAyrnKBcBjwIdEfHFkqeeBlamj1cCa0a/vCrgKy+aWY0oZw/9auATwLWS1qfTbwIPADdI2gJc\nny7nz2WXweTJHhg1s6o3frANIuKfAfXz9HWjW04VamyE5cu9h25mVc9nipajrQ3WrUvuYmRmVqUc\n6OUoFOCXv4SNG7OuxMysXw70cnhg1MxqgAO9HIsWwYwZHhg1s6rmQC+HlOylO9DNrIo50MtVKMCG\nDUkv3cysCjnQy9XWBqdOwfr1WVdiZtYnB3q5igOjP/xhtnWYmfXDgV6uefPgkkvg3nvh2mvhG99w\n+8XMqooDfSheeAE+/3nYvh0+8QmYOxd+7/d8NUYzqwoO9KE491z4kz+BLVvgxRfh5pvhr/86accs\nXQoPPQRvv511lWZWpxzowzFuHFxzDfzt38KePfDlL8OkSfDpTyetmY9+FJ59NhlENTMbIw70kTrr\nLPjUp5KzSDdsgLvvTvbeb7wRWlrgc5/ztdTNbEw40EfTFVfAgw/Crl3wne8ky3/+53DhhR5INbOK\nc6BXwsSJcPvt8L3vJQOoHkg1szHgQK+0BQs8kGpmY8KBPlZ6D6Q+/HD3QOr8+XDHHR5INbMRcaBn\n4ayz4Hd/t3sg9fd/PznG3QOpZjYCDvSs9R5Ivfxy+LM/6x5IfewxD6SaWVkc6NWiOJD6/e/3HEi9\n804PpJpZWRzo1ei88/ofSF22LBlI3bjR/XYz60Exhnt8ra2t0d7ePmbvlysHD8Ljj8PXvtZ9o42m\nJli+HK66KplaW+Gii6ChIdtazWxUSVobEa2DbjdYoEv6GnATsC8iLk/XzQKeAFqAbcAdEXFgsDdz\noI+SzZvhxz+GtWuT6ZVXuvvsxZBvbe0Oeoe8WU0bzUD/IHAU+JuSQP9vwP6IeEDS/cDMiLhvsDdz\noFdIVxds2tQd8O3tyY04SkN+xYrugHfIm9WUUQv09Ie1AM+UBPpm4JqI2C1pLvCDiFgy2M9xoI+h\n0pBvb0/mpSE/dWrPdo1D3qxqVTrQD0bEjPSxgAPF5T5euwpYBbBw4cKrtm/fXu7vYKOtqws6Orr3\n5AcK+WLL5qKLkpOizCwzYxbo6fKBiJg52M/xHnoV6h3yxXbNe+8lzxdDvndP3iFvNmbKDfTxw/z5\neyXNLWm57Bvmz7GsjR+fnNx0xRXwyU8m60pDvtiuefjhniG/YkUyLVkCixcn04IFDnqzDA030J8G\nVgIPpPM1o1aRZa+/kN+4sWe75itf6XkW68SJyRmuxYBfvBgWLUrm8+c77M0qrJyjXL4FXAPMBvYC\nfwr8T+DbwEJgO8lhi/sHezO3XHLm9OnkkgVbtnRPb7yRzH/+czh+vHvbSZP6D/t58xz2ZgMY1R76\naHGg15HTp2Hnzv7D/sSJ7m0nT07CvRjwpYE/bx5I2f0eZlWg0j10s4GNGwcLFybTddf1fO7Uqb7D\nvqMDvvvdnmE/ZUp30JcG/qJFyTVuHPZmv+JAt7HX0ADnn59M11/f87lTp2DHjjPD/vXX4emn4eTJ\n7m2bmpJgv/DCpEc/b96Z01lnOfStbjjQrbo0NCTXhG9pgRtu6PlcV1fPsC+2cDo64Pnn4dChM3/e\n5Ml9B33vaerUsfjtzCrKPXTLj2PHYPdu+MUv+p927YJ33z3ztdOmDR76c+cmXxBmY8w9dKs/xRbM\nokX9bxMBR44MHPo/+lEyLz1Kp2jmzP4Df84cmDUr2WbmzOQwTrMx5EC3+iLB9OnJdPHF/W8XAQcO\n9L2HX3zc0ZH8RdDfdeknT06CvTTk+1rua11jY2V+f8s1B7pZX6QkZGfNSm4L2J/Tp6GzMwn4ffuS\nL4H9+5N5cSoub9uWXOr4wAE4enTg929qGt4XwYwZyYlhVpf8X95sJMaNS1otc+YM7XUnTiQ3Lekr\n+Pta98Yb3ev6GgMoNW1acnTPjBnJfKiPm5p8ZFCNcqCbZWHCBDjnnGQaquPH+w/+AweSL4pDh7rn\ne/cmN0UpruvqGvjnNzQM74ug9LHHDzLhQDerNRMnwrnnJtNQRSTX3zl0qGfo9/e4OH/zze7Hhw8P\nfrPyiROTcYpp04Y29fUa/8VQNge6WT2RkrNvp0xJDsMcjtOnkyOFBvsiOHKk59TZCVu3di8fPTr4\nF0Ox5qlTh/7lUPqFUJymTEnmjY25/JJwoJvZ0Iwb191iGYnTp5PxgNLQP3z4zC+C/qbt23suFy/v\nXI6Ghp4BP9h8KNtOmZIc4ZTBF4YD3cyyMW5csuc9derw/1oodfJkstffO/iPHUu+OMqZHzuWjEP0\nfq6vcxIG0zvo16wZ+ByJUeBAN7N8aGzsPnxztHV1JWMP5X4p9PVcU9Po19WLA93MbDDjx3f35KuY\n7ypgZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McmJM7ykqqRPYPsyXzwbe\nHsVyap0/j27+LHry59FTHj6P8yOiebCNxjTQR0JSezk3Sa0X/jy6+bPoyZ9HT/X0ebjlYmaWEw50\nM7OcqKVAfyTrAqqMP49u/ix68ufRU918HjXTQzczs4HV0h66mZkNwIFuZpYTNRHokj4kabOkNyTd\nn3U9WZF0nqQXJW2U9FNJ92RdUzWQ1CDpFUnPZF1L1iTNkPSkpE2SOiR9IOuasiLpM+m/k9clfUvS\npKxrqrSqD3RJDcCXgBuBS4HflnRptlVlpgv4bERcCrwfuLuOP4tS9wAdWRdRJR4Cno2Ii4Gl1Onn\nImk+8O+B1oi4HGgAPpZtVZVX9YEOtAFvRMTWiDgBPA7cmnFNmYiI3RGxLn18hOQf6/xsq8qWpAXA\nh4GvZl1L1iSdBXwQeBQgIk5ExMFsq8rUeGCypPHAFOAXGddTcbUQ6POBHSXLO6nzEAOQ1AIsB17K\ntpLM/SXwR8DprAupAhcAncDX0xbUVyVV/s7EVSgidgH/HXgL2A0cioh/zLaqyquFQLdeJE0F/g74\ndEQczrqerEi6CdgXEWuzrqVKjAdWAA9HxHLgGFCXY06SZpL8JX8BMA9oknRntlVVXi0E+i7gvJLl\nBem6uiSpkSTMH4uIp7KuJ2NXA7dI2kbSirtW0jeyLSlTO4GdEVH8q+1JkoCvR9cDb0ZEZ0ScBJ4C\n/kXGNVVcLQT6y8BiSRdImkAysPF0xjVlQpJI+qMdEfHFrOvJWkT8cUQsiIgWkv8vXoiI3O+F9Sci\n9gA7JC1JV10HbMywpCy9Bbxf0pT038111MEA8fisCxhMRHRJ+nfA/yIZqf5aRPw047KycjXwCeA1\nSevTdf8hIr6XYU1WXf4AeCzd+dkK/E7G9WQiIl6S9CSwjuTosFeog0sA+NR/M7OcqIWWi5mZlcGB\nbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLif8Pko9BNoG0/DQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt0nPV95/H3RzdLlmVZxvIFX+QL\nJsaEi0HHDaVJSViIY5pQQpt1mmSTnqbs9pRsmtBuSU9LsmS7ydn29LbL6YambDdJCeVAt/UWJ2wS\nIElT0tgzNhAbML5gZHsMAt9tWdfv/jGP7LEsWyN75Eea+bzO0dHz/J7fM/OdAX308+/3PBpFBGZm\nVhmq0i7AzMwuHoe+mVkFceibmVUQh76ZWQVx6JuZVRCHvplZBXHo24QgqVrSUUkLStnXrNI49G1M\nJKE7+DUgqatg/yOjfbyI6I+IKRHxWin7ni9Jn5QUku4cq+cwGwvyzVk21iS9CnwyIr57jj41EdF3\n8aq6MJJ+CCwH/jkibr/Iz10dEf0X8zmtfHikb6mQ9F8k/Z2kb0o6AnxU0g2SfizpoKScpL+QVJv0\nr0lG1guT/W8kx78l6YikZyUtGm3f5Pj7JG2VdEjSf5f0I0mfOEftS4AbgbuA90lqHXL8g5I2STos\naZukW5P2SyT9TfLaDkh6PGn/pKRnCs4frv4HJH1b0jHgnZI+UPAcr0n6gyE1vCt5Lw9J6pD0seT9\n3SupqqDfhyRlRvGfziY4h76l6Q7gYaAZ+DugD/g0MIN8qK4C/v05zv8V4A+A6cBrwBdH21fSTOBR\n4HeS590JrByh7n8H/DgiHge2J49N8ng/CzwE3ANMA94N7EoOPwzUkf8Xwkzgz0d4nqH1/2egCXgW\nOAp8JHmO9wOflvQLSQ2LgHXAnwCXACuAFyLiWeAIcHPB434M+Noo6rAJzqFvafrniPi/ETEQEV0R\nsT4i/jUi+iJiB/Ag8PPnOP+xiNgQEb3A3wLXnkffXwA2RcQ/Jsf+FHjzbA8iSeRD/+Gk6eFkf9Cv\nAX8VEd9LXldHRLwsaT75sP2NiDgQEb0R8YNz1DvU/4mIZ5PH7I6IpyJic7L/HPAIp96rjwLfiohH\nk/fyzYjYlBz7WnIcSTOSmr45ijpsgnPoW5o6CnckLZP0hKR9kg4D95MffZ/NvoLt48CU8+h7aWEd\nkV/k2n2Ox3kXMI/8v0wgH/rXSXp7sj+f/Oh/qPnAmxFx6ByPfS5D36sbJD0jqVPSIeCTnHqvzlYD\nwNeB2yU1AGuApyPijfOsySYgh76laehVBF8BfgpcFhFTgfsAjXENOfIhDpwcyc89R/+Pk/+5eUHS\nPuBH5F/Hx5PjHcCSYc7rAGZImjrMsWPA5IL92cP0GfpePQI8DsyPiGbgq5x6r85WA8kVTRngF8lP\n7Xx9uH5Wvhz6Np40AYeAY5Ku4Nzz+aXyT+RH6u+XVEN+TaF1uI6SJgO/RH4K59qCr88AH5FUDfw1\n8ElJ75ZUJWmepLdFRAfwXeABSdMk1Up6V/LQzwFXS7oqGYF/voi6m4D9EXFC0jvIj9oHfQNYJenO\nZFF4hqRrCo5/DfgcsAz4xyKey8qIQ9/Gk3vIj5iPkB/1/925u1+4iHgd+LfkFz3fIj9C3gh0D9P9\ng0lt34iIfYNfwF8BDcAtEfEvwK8Df0H+F9jT5KdbIJlLB7YCrwOfSmrYAvxX4BngZaCYuf7fAL6U\nXPn0e+QXowdf007yi7u/C+wHssBVBec+Diwmv87RVcRzWRnxdfpmBZLR+l7glyLih2nXMxaSKayd\nwCci4pmUy7GLzCN9q3iSViVTLpPIX9bZC/wk5bLG0ofI/0vm+2kXYhdfTdoFmI0DP0f+KpwaYDNw\nR0QMN70z4Un6Z2Ap8JHwP/Mrkqd3zMwqiKd3zMwqyLib3pkxY0YsXLgw7TLMzCaUTCbzZkQMe7lx\noXEX+gsXLmTDhg1pl2FmNqFI2jVyL0/vmJlVFIe+mVkFceibmVUQh76ZWQVx6JuZVRCHvplZBXHo\nm5lVkHF3nb6ZWTnoH+jneO/x076O9R47s63nVNusKbO46/q7xrQuh76ZVZSIoLu/m67eLrr6ujje\ne5yu3q4Rw/m0gO4bPrQLz+/p7xl1be+Y9w6HvpmVt4igp78nH759XSfDeGgoDz12Rv+ztQ95nBN9\nJ4gzPn1yZJOqJ9FY18jk2smnfTXWNTKzceYZ7SeP1555TuG5g9sNNQ3UVteOwTt8Ooe+mY1ocHR8\nrOcYx3qPcbTn6MntYz3J/rm2h5xTuN3V23VeIQxQV11HQ00DDbUNJ4OzobaBhpoGmuubmVM7J99W\n0D5c34bahpPhPFywN9Q0UF1VXeJ3NR0OfbMy0zfQx9GeoxztOcqR7iP57z1HTu4Pbp8RwiMEeH/0\nF11DlaporG1kSt0UGusaT2431zdzadOlNNY1MqV2yqlQTcK3cHuktvqa+rIJ4ovJoW+Wsu6+7tNC\neTCkzxXYw7Yl/bv6iv/Y24aahjOCubGukekN0/PbtY1nHDvXdmNdfn9S9STyn8po441D3+wC9fT3\ncKDrAPu79rO/az8HTuS3B9tO7iffD3cfPi2kewd6i3qealXTNKmJpromptRNoWlS/vuMyTPy27Wn\n2prqmk7bHuw/uD0Y1FXyVduVxqFvBgzEAIe7Dw8b1ucK8ANdBzjWe+ysjytEc30z0xum01LfQktD\nC3Ob5o46pJsmNXn0bCXh0LeyFBEcPHGQXYd2sevgLl479BqvH3s9H94n9p8xMj944iADMXDWx2uo\naaCloeVkeC+atojr5lzH9Prp+baCY9MbTrU1T2r2vLONKw59m5AGYoB9R/ex6+Cu04J916FT+0d6\njpx2TpWqTo62B4P5sumXnRHUheHd0tBCS30LDbUNKb1Ss9Jy6Nu41NPfQ8ehjrMGesfhjjNufplW\nP4225jYWtyzm3QvfTVtzG23T2ljQvIC25jZaG1s9h20Vz6FvqTjSfeRkgO86VBDqyX7uSO6Ma7fn\nTJlD27Q22i9t584r7qRtWttpwT510tSUXo3ZxOHQtzFxpPsIOw7sYPuB7ew8sPOMkfqBEwdO619b\nVcv85vm0Nbdx65Jb82GeBHpbcxvzps5jUs2klF6NWflw6Nt5iQj2Hd3H9gPb2b5/+8mAH9zvPN55\nWv+muqaTI/Ib5t1wWqC3TWtj9pTZnnoxuwgc+nZW3X3d7Dq0i+3782F+MtiTkC+8CahKVcyfOp/F\nLYu5/W23s2T6Eha3LGZJS/77tPppvtzQbBxw6Fe4A10HzgjzwRF7x6GO0+bVG2oaWDJ9CUumL+HW\nJbeeDPQl05ewcNpC6qrrUnwlZlaMokJf0irgz4Fq4KsR8eUhx9uAh4BWYD/w0YjYnRzrB15Iur4W\nER8oUe1WhIEYYPfh3fkwH2bEPnRufWbjTJa0LOGdC97JkpYlp43YZ0+Z7dG62QQ3YuhLqgYeAG4B\ndgPrJa2NiC0F3f4Y+FpE/G9J7wG+BHwsOdYVEdeWuG47iyPdR/juju/yxCtP8KOOH7HjwI7TLm2s\nqaqhrbmNJdOX0H5le37knozYF7cspmlSU4rVm9lYK2akvxLYFhE7ACQ9AtwOFIb+cuCzyfbTwD+U\nskg7u4hg61tbWffKOp545Ql+sOsH9A70MnXSVG5aeBPvv/z9p43YFzQvoKbKs3pmlaqYn/65QEfB\n/m7gZ4b0eQ74IPkpoDuAJkmXRMRbQL2kDUAf8OWIOOMXgqS7gLsAFixYMOoXUWlO9J3g+69+nyde\neYJ1r6xj+4HtACxvXc5vveO3WL10NTfOv/GifCCDmU0spRry/TbwPyR9AvgBsAcY/OPbbRGxR9Ji\n4ClJL0TE9sKTI+JB4EGA9vb28/s0hTLXcajj5Gj+ezu/x/He49TX1POeRe/hszd8ltVLV7Nw2sK0\nyzSzca6Y0N8DzC/Yn5e0nRQRe8mP9JE0BbgzIg4mx/Yk33dIegZYAZwW+namvoE+nu149uRo/oU3\n8mvhC6ct5Fev/VVWL13Nuxe+238TxsxGpZjQXw8slbSIfNivAX6lsIOkGcD+iBgAPkf+Sh4ktQDH\nI6I76XMj8N9KWH9Z6TzWybe3fZsnXnmCJ7c/ycETB6mpquHnFvwcf3TLH3Hb0ttYNmOZr6Axs/M2\nYuhHRJ+ku4EnyV+y+VBEbJZ0P7AhItYCNwFfkhTkp3d+Mzn9CuArkgaAKvJz+lvOeJIKNRADbMxt\nPDlt85M9PyEIZjXO4o5ld7B66WpuWXwLzfXNaZdqZmVCEeNrCr29vT02bNiQdhlj5nD3Yb6z/Ts8\n8coTfGvbt9h3dB9CrJy7ktVLV3Pb0ttYMWeF/ySBmY2KpExEtI/Uz9fujbGI4KU3Xzo5N//D135I\n30Af0+qn8d4l72X10tWsumwVMxtnpl2qmVUAh/4Y6Ort4plXnzkZ9DsP7gTgqplXcc8N93Db0tu4\nYf4Nvl7ezC46p06Jvdj5Iiu/upKjPUeZXDuZmxfdzO/e+Lu8b+n7WNDsexDMLF0O/RL73s7vcbTn\nKI9/6HFWL11NfU192iWZmZ3k0C+xbC7LzMaZ3LHsDl9aaWbjji8RKbFMLsN1c65z4JvZuOTQL6Gu\n3i42v7GZ6+dcn3YpZmbDcuiX0AtvvEB/9HPdnOvSLsXMbFgO/RLK7M0AeKRvZuOWQ7+Esrks0xum\n+9JMMxu3HPollMlluH7O9V7ENbNxy6FfIt193fz0jZ96asfMxjWHfon89I2f0jvQ60VcMxvXHPol\nkskli7iXeqRvZuOXQ79Esrks0+qnsWjaorRLMTM7K4d+ifhOXDObCBz6JdDT38Pzrz/vRVwzG/cc\n+iWwpXMLPf09XsQ1s3HPoV8CvhPXzCaKokJf0ipJL0vaJuneYY63SfqepOclPSNpXsGxj0t6Jfn6\neCmLHy+yuSxNdU0smb4k7VLMzM5pxNCXVA08ALwPWA58WNLyId3+GPhaRFwN3A98KTl3OvB54GeA\nlcDnJbWUrvzxYXAR1x9mbmbjXTEptRLYFhE7IqIHeAS4fUif5cBTyfbTBcffC3wnIvZHxAHgO8Cq\nCy97/Ogb6OO515/z1I6ZTQjFhP5coKNgf3fSVug54IPJ9h1Ak6RLijwXSXdJ2iBpQ2dnZ7G1jwsv\ndr7Iib4TXsQ1swmhVPMRvw38vKSNwM8De4D+Yk+OiAcjoj0i2ltbW0tU0sXhO3HNbCIp5jNy9wDz\nC/bnJW0nRcRekpG+pCnAnRFxUNIe4KYh5z5zAfWOO9lclsbaRpZOX5p2KWZmIypmpL8eWCppkaQ6\nYA2wtrCDpBnSyVXMzwEPJdtPArdKakkWcG9N2spGJpdhxZwVVFdVp12KmdmIRgz9iOgD7iYf1i8C\nj0bEZkn3S/pA0u0m4GVJW4FZwB8m5+4Hvkj+F8d64P6krSz0D/Szad8mL+Ka2YRRzPQOEbEOWDek\n7b6C7ceAx85y7kOcGvmXlZffepnjvce9iGtmE4YvLL8AvhPXzCYah/4FyOayNNQ08LYZb0u7FDOz\nojj0L0Aml+Ha2ddSU1XULJmZWeoc+udpIAbYuG+jp3bMbEJx6J+nV956haM9R72Ia2YTikP/PPlO\nXDObiBz65ymbyzKpehJXzLgi7VLMzIrm0D9PmVyGa2ZfQ211bdqlmJkVzaF/HgZigGwuy3WzPZ9v\nZhOLQ/887Diwg8Pdhz2fb2YTjkP/PPhOXDObqBz65yGby1JXXceVM69MuxQzs1Fx6J+HTC7DVTOv\noq66Lu1SzMxGxaE/ShGRX8T1TVlmNgE59Efp1YOvcuDEAc/nm9mE5NAfJd+Ja2YTmUN/lLK5LDVV\nNbx95tvTLsXMbNQc+qOUyWV4+8y3U19Tn3YpZmaj5tAfhZOLuL4T18wmqKJCX9IqSS9L2ibp3mGO\nL5D0tKSNkp6XtDppXyipS9Km5Ot/lvoFXEwdhzt48/ibns83swlrxI98klQNPADcAuwG1ktaGxFb\nCrr9PvBoRPylpOXkP0R9YXJse0RcW9qy0+E7cc1soitmpL8S2BYROyKiB3gEuH1InwCmJtvNwN7S\nlTh+ZHNZqlXN1bOuTrsUM7PzUkzozwU6CvZ3J22FvgB8VNJu8qP8TxUcW5RM+3xf0juHewJJd0na\nIGlDZ2dn8dVfZJlchuWty2mobUi7FDOz81KqhdwPA38TEfOA1cDXJVUBOWBBRKwAPgs8LGnq0JMj\n4sGIaI+I9tbW1hKVVFoRQSaX8Z24ZjahFRP6e4D5BfvzkrZCvwY8ChARzwL1wIyI6I6It5L2DLAd\nuPxCi07D3iN7eePYG57PN7MJrZjQXw8slbRIUh2wBlg7pM9rwM0Akq4gH/qdklqThWAkLQaWAjtK\nVfzF5DtxzawcjHj1TkT0SbobeBKoBh6KiM2S7gc2RMRa4B7gryR9hvyi7iciIiS9C7hfUi8wAPyH\niNg/Zq9mDGVzWapUxTWzrkm7FDOz8zZi6ANExDryC7SFbfcVbG8BbhzmvMeBxy+wxnEhk8uwbMYy\nGusa0y7FzOy8+Y7cIvnPKZtZOXDoF2Hf0X3sPbLXi7hmNuE59IvgO3HNrFw49IuQzWUR4trZZfHX\nJMysgjn0i5DJZbj8kstpmtSUdilmZhfEoV8EL+KaWblw6I+g81gnHYc7PJ9vZmXBoT+CwTtxPdI3\ns3Lg0B9BNpcFHPpmVh4c+iPI5DJcNv0ymuub0y7FzOyCOfRH4EVcMysnDv1zeOv4W7x68FUv4ppZ\n2XDon8PGfRsBz+ebWflw6J/D4J9fcOibWblw6J9DJpdh0bRFTG+YnnYpZmYl4dA/By/imlm5ceif\nxcETB9l+YLsXcc2srDj0z2Jjzou4ZlZ+HPpn4Q9CN7NyVFToS1ol6WVJ2yTdO8zxBZKelrRR0vOS\nVhcc+1xy3suS3lvK4sdSJpdhQfMCZkyekXYpZmYlM+IHo0uqBh4AbgF2A+slrU0+DH3Q7wOPRsRf\nSlpO/kPUFybba4ArgUuB70q6PCL6S/1CSs2LuGZWjooZ6a8EtkXEjojoAR4Bbh/SJ4CpyXYzsDfZ\nvh14JCK6I2InsC15vHHtcPdhtr611Yu4ZlZ2ign9uUBHwf7upK3QF4CPStpNfpT/qVGci6S7JG2Q\ntKGzs7PI0sfOpn2bAC/imln5KdVC7oeBv4mIecBq4OuSin7siHgwItojor21tbVEJZ0/fxC6mZWr\nEef0gT3A/IL9eUlboV8DVgFExLOS6oEZRZ477mRyGeY2zWXWlFlpl2JmVlLFjMbXA0slLZJUR35h\ndu2QPq8BNwNIugKoBzqTfmskTZK0CFgK/KRUxY8VL+KaWbkaMfQjog+4G3gSeJH8VTqbJd0v6QNJ\nt3uAX5f0HPBN4BORtxl4FNgCfBv4zfF+5c7RnqO89OZLntoxs7JUzPQOEbGO/AJtYdt9BdtbgBvP\ncu4fAn94ATVeVM/te44gPNI3s7LkO3KH8J24ZlbOHPpDZHIZZjXOYs6UOWmXYmZWcg79IbK5LNdf\nej2S0i7FzKzkHPoFjvceZ0vnFi/imlnZcugXeP715xmIAS/imlnZcugX8J24ZlbuHPoFMrkMMybP\nYN7UeWmXYmY2Jhz6BbK5LNfP8SKumZUvh37iRN8JNndu9tSOmZU1h37ihddfoG+gz4u4ZlbWHPoJ\n34lrZpXAoZ/I7M3QUt9CW3Nb2qWYmY0Zh34iu8934ppZ+XPoA9193bzw+gtexDWzsufQBzZ3bqZ3\noNeLuGZW9hz6+E5cM6scDn3yN2U1T2pmccvitEsxMxtTDn3yl2teN+c6L+KaWdmr+NDv7e/l+def\n99SOmVWEokJf0ipJL0vaJuneYY7/qaRNyddWSQcLjvUXHFtbyuJLYUvnFrr7u72Ia2YVYcQPRpdU\nDTwA3ALsBtZLWpt8GDoAEfGZgv6fAlYUPERXRFxbupJLy3fimlklKWakvxLYFhE7IqIHeAS4/Rz9\nPwx8sxTFXQzZXJamuiYum35Z2qWYmY25YkJ/LtBRsL87aTuDpDZgEfBUQXO9pA2SfizpF8+70jGS\nyWVYMWcFVar45Q0zqwClTro1wGMR0V/Q1hYR7cCvAH8macnQkyTdlfxi2NDZ2Vniks6ub6CP5/Y9\n50VcM6sYxYT+HmB+wf68pG04axgytRMRe5LvO4BnOH2+f7DPgxHRHhHtra2tRZRUGi+9+RJdfV1e\nxDWzilFM6K8HlkpaJKmOfLCfcRWOpGVAC/BsQVuLpEnJ9gzgRmDL0HPT4jtxzazSjHj1TkT0Sbob\neBKoBh6KiM2S7gc2RMTgL4A1wCMREQWnXwF8RdIA+V8wXy686idt2VyWxtpGLr/k8rRLMTO7KEYM\nfYCIWAesG9J235D9Lwxz3r8AV11AfWMqk8tw7exrqa6qTrsUM7OLomIvWekf6Gfjvo2ezzezilKx\nob/1ra0c7z3u+XwzqygVG/q+E9fMKlHFhn42l6WhpoFlM5alXYqZ2UVTsaGfyWW4ZvY11FQVtZZt\nZlYWKjL0B2KAjbmNXDfbi7hmVlkqMvS37d/GkZ4jns83s4pTkaHvO3HNrFJVZOhnc1kmVU9ieevy\ntEsxM7uoKjL0M7kMV8+6mtrq2rRLMTO7qCou9COCbC7rO3HNrCJVXOjvOLCDQ92HPJ9vZhWp4kLf\nd+KaWSWruNDP5rLUVtVyZeuVaZdiZnbRVVzoZ3IZrpp1FZNqJqVdipnZRVdRoX9yEdd34ppZhaqo\n0N91aBf7u/Z7Pt/MKlZFhb7vxDWzSldRoZ/NZampquGqWeP2ExzNzMZURYV+JpfhytYrqa+pT7sU\nM7NUFBX6klZJelnSNkn3DnP8TyVtSr62SjpYcOzjkl5Jvj5eyuJHw3fimpnBiJ8gIqkaeAC4BdgN\nrJe0NiK2DPaJiM8U9P8UsCLZng58HmgHAsgk5x4o6asowu7Du+k83un5fDOraMWM9FcC2yJiR0T0\nAI8At5+j/4eBbybb7wW+ExH7k6D/DrDqQgo+X74T18ysuNCfC3QU7O9O2s4gqQ1YBDw1mnMl3SVp\ng6QNnZ2dxdQ9atlclipVcfWsq8fk8c3MJoJSL+SuAR6LiP7RnBQRD0ZEe0S0t7a2lrikvEwuw/LW\n5UyunTwmj29mNhEUE/p7gPkF+/OStuGs4dTUzmjPHVNexDUzKy701wNLJS2SVEc+2NcO7SRpGdAC\nPFvQ/CRwq6QWSS3ArUnbRbX3yF72Hd3nRVwzq3gjXr0TEX2S7iYf1tXAQxGxWdL9wIaIGPwFsAZ4\nJCKi4Nz9kr5I/hcHwP0Rsb+0L2Fkg3fieqRvZpVuxNAHiIh1wLohbfcN2f/CWc59CHjoPOsriWwu\nixDXzr42zTLMzFJXEXfkZnIZls1YxpS6KWmXYmaWqooIfS/impnllX3ov370dfYc2eNFXDMzKiD0\nB+/E9UjfzKwCQj+bywKwYs6KlCsxM0tf2Yd+Jpfh8ksuZ+qkqWmXYmaWurIPfS/impmdUtah/+bx\nN3nt0GtexDUzS5R16PtOXDOz05V16A8u4jr0zczyyjr0M7kMS1qWMK1+WtqlmJmNC2Ud+l7ENTM7\nXdmG/v6u/ew8uNOLuGZmBco29D2fb2Z2Joe+mVkFKdvQz+QyLJy2kEsmX5J2KWZm40bZhr4Xcc3M\nzlSWoX/oxCG27d/mRVwzsyHKMvQ37tsIeD7fzGyookJf0ipJL0vaJunes/T5kKQtkjZLerigvV/S\npuRr7XDnlpr//IKZ2fBG/GB0SdXAA8AtwG5gvaS1EbGloM9S4HPAjRFxQNLMgofoioiL+onkmVyG\neVPnMbNx5sidzcwqSDEj/ZXAtojYERE9wCPA7UP6/DrwQEQcAIiIN0pb5uhkc1nP55uZDaOY0J8L\ndBTs707aCl0OXC7pR5J+LGlVwbF6SRuS9l8c7gkk3ZX02dDZ2TmqFzDUke4jbH1rq0PfzGwYI07v\njOJxlgI3AfOAH0i6KiIOAm0RsUfSYuApSS9ExPbCkyPiQeBBgPb29riQQjbt20QQns83MxtGMSP9\nPcD8gv15SVuh3cDaiOiNiJ3AVvK/BIiIPcn3HcAzwJh+WO3gB6Fff6lH+mZmQxUT+uuBpZIWSaoD\n1gBDr8L5B/KjfCTNID/ds0NSi6RJBe03AlsYQ5lchjlT5jB7yuyxfBozswlpxOmdiOiTdDfwJFAN\nPBQRmyXdD2yIiLXJsVslbQH6gd+JiLck/SzwFUkD5H/BfLnwqp+xkM1lPco3MzuLoub0I2IdsG5I\n230F2wF8Nvkq7PMvwFUXXmZxjvUc46U3X+KXl//yxXpKM7MJpazuyH3u9ecYiAEv4pqZnUVZhf7g\nnbi+XNPMbHjlFfq5DDMbZ3Jp06Vpl2JmNi6VVegP3okrKe1SzMzGpbIJ/a7eLrZ0bvHUjpnZOZRN\n6B/uPsyHrvwQNy28Ke1SzMzGrVL9GYbUzZoyi4fvfHjkjmZmFaxsRvpmZjYyh76ZWQVx6JuZVRCH\nvplZBXHom5lVEIe+mVkFceibmVUQh76ZWQVR/k/hjx+SOoFdF/AQM4A3S1TOROf34nR+P07n9+OU\ncngv2iKidaRO4y70L5SkDRHRnnYd44Hfi9P5/Tid349TKum98PSOmVkFceibmVWQcgz9B9MuYBzx\ne3E6vx+n8/txSsW8F2U3p29mZmdXjiN9MzM7C4e+mVkFKZvQl7RK0suStkm6N+160iRpvqSnJW2R\ntFnSp9OuKW2SqiVtlPRPadeSNknTJD0m6SVJL0q6Ie2a0iTpM8nPyU8lfVNSfdo1jaWyCH1J1cAD\nwPuA5cCHJS1Pt6pU9QH3RMRy4B3Ab1b4+wHwaeDFtIsYJ/4c+HZELAOuoYLfF0lzgf8ItEfE24Fq\nYE26VY2tsgh9YCWwLSJ2REQP8Ahwe8o1pSYichGRTbaPkP+hnptuVemRNA+4Dfhq2rWkTVIz8C7g\nrwEioiciDqZbVepqgAZJNcBkYG/K9Yypcgn9uUBHwf5uKjjkCklaCKwA/jXdSlL1Z8B/AgbSLmQc\nWAR0Av8rme76qqTGtItKS0TsAf4YeA3IAYci4v+lW9XYKpfQt2FImgI8DvxWRBxOu540SPoF4I2I\nyKRdyzhRA1wH/GVErACOARWBxErUAAABJ0lEQVS7BiaphfyswCLgUqBR0kfTrWpslUvo7wHmF+zP\nS9oqlqRa8oH/txHx92nXk6IbgQ9IepX8tN97JH0j3ZJStRvYHRGD//J7jPwvgUr1b4CdEdEZEb3A\n3wM/m3JNY6pcQn89sFTSIkl15Bdi1qZcU2okifyc7YsR8Sdp15OmiPhcRMyLiIXk/794KiLKeiR3\nLhGxD+iQ9Lak6WZgS4olpe014B2SJic/NzdT5gvbNWkXUAoR0SfpbuBJ8qvvD0XE5pTLStONwMeA\nFyRtStp+LyLWpViTjR+fAv42GSDtAH415XpSExH/KukxIEv+qreNlPmfZPCfYTAzqyDlMr1jZmZF\ncOibmVUQh76ZWQVx6JuZVRCHvplZBXHom5lVEIe+mVkF+f+cOMdJ4rySewAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X4WYxO2_0gbq",
        "colab_type": "text"
      },
      "source": [
        "##Testing the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4om_ZyHT6ALB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9648dc87-a441-492a-e6bb-bf1eee74d8fa"
      },
      "source": [
        "acc = 0.0\n",
        "batches = 0\n",
        "for features, labels in test_data:\n",
        "    features = features.as_in_context(ctx)\n",
        "    labels = labels.as_in_context(ctx).one_hot(10)\n",
        "    predictions = model(features)\n",
        "    acc += accuracy(predictions, labels)\n",
        "    batches += 1\n",
        "print(f'Test Accuracy: {acc.asscalar()/batches}')"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Accuracy: 0.9685509554140127\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}