convNet_with_3x3_kernels.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "convNet with 3x3 kernels",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guUo5q-RA9Ws",
        "colab_type": "text"
      },
      "source": [
        "Applying Convolutional neural network on MNIST dataset (In this notebook, CNN is tried out with different number of layers and different architectures with help of 3x3 kernel)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ao5cTwHrCCf8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#loading all the necessary packages for data processing and plotting\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import plotly.express as px\n",
        "import plotly as py\n",
        "import plotly.graph_objects as go\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vV46xaIPGOJZ",
        "colab_type": "code",
        "outputId": "eecff340-f162-45a1-b2a8-546f5b6995db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "#importing all the deep_learning packages\n",
        "import tensorflow.keras \n",
        "from tensorflow.keras.layers import Dense,Activation,Conv2D,MaxPooling2D,Dropout,Flatten\n",
        "from tensorflow.keras.models import Sequential\n",
        "from keras.utils import np_utils\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.initializers import RandomNormal,RandomUniform,he_normal,he_uniform\n",
        "from tensorflow.keras import backend as k"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cHHUYajpGzaM",
        "colab_type": "code",
        "outputId": "0244619e-64a2-4a0c-b809-c2719dd855a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "#load the data and make the data ready to pass on to the architecture\n",
        "(X_train,y_train),(X_test,y_test) = mnist.load_data()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V-9UJ38TJIw6",
        "colab_type": "code",
        "outputId": "e1795a7f-6fb9-4e99-9636-33eeee1bc312",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "print(f\"train data shape {X_train.shape}\")\n",
        "print(f\"test data shape {X_test.shape}\")\n",
        "print(f\"train data labels shape {y_train.shape}\")\n",
        "print(f\"test data labels shape {y_test.shape}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train data shape (60000, 28, 28)\n",
            "test data shape (10000, 28, 28)\n",
            "train data labels shape (60000,)\n",
            "test data labels shape (10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8wc13JCfm8u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img_rows,img_cols = 28,28\n",
        "\n",
        "if tensorflow.keras.backend.image_data_format() == 'channels_first':\n",
        "    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
        "    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
        "    input_shape = (1, img_rows, img_cols)\n",
        "else:\n",
        "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
        "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
        "    input_shape = (img_rows, img_cols, 1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xx-qDZwhTwzs",
        "colab_type": "code",
        "outputId": "a39ea7be-90e9-4383-fc55-aa4901eed9d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "#normalizing the data\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "\n",
        "X_train = (X_train-X_train.min())/(X_train.max()-X_train.min())\n",
        "X_test = (X_test-X_test.min())/(X_test.max()-X_test.min())\n",
        "\n",
        "print(f\"train samples -- {X_train.shape[0]}\")\n",
        "print(f\"test samples -- {X_test.shape[0]}\")\n",
        "\n",
        "print(f\"train samples shape -- {X_train.shape}\")\n",
        "print(f\"test samples shape -- {X_test.shape}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train samples -- 60000\n",
            "test samples -- 10000\n",
            "train samples shape -- (60000, 28, 28, 1)\n",
            "test samples shape -- (10000, 28, 28, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AbVtEzcnL2qp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#other important parameters\n",
        "batch_size = 128\n",
        "epochs = 20"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZgO2Nv-L2ip",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# we have a class number for each image\n",
        "# lets convert this into a 10 dimensional vector\n",
        "# ex: consider an image is 5 convert it into 5 => [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]\n",
        "# this conversion needed for MLPs\n",
        "Y_train = np_utils.to_categorical(y_train, 10)\n",
        "Y_test = np_utils.to_categorical(y_test, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_QxXnxKjUN77",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#plot function to generate loss plots\n",
        "no_of_epochs = list(range(1,21))\n",
        "\n",
        "\n",
        "def loss_plots(n_epochs,train_loss,validation_loss):\n",
        "\n",
        "  fig = go.Figure()\n",
        "  fig.add_trace(go.Scatter(x=n_epochs, y=train_loss,\n",
        "                  mode='lines+markers',name='train_loss'))\n",
        "\n",
        "  fig.add_trace(go.Scatter(x=n_epochs, y=validation_loss,\n",
        "                  mode='lines+markers',name='validation_loss'))\n",
        "\n",
        "  fig.update_layout(title='Train_loss and Validation_loss',xaxis_title='number of epochs',yaxis_title='Categorical Cross Entropy Loss')\n",
        "  fig.show()\n",
        "\n",
        "def accuracy_plots(n_epochs,train_accuracy,validation_accuracy):\n",
        "\n",
        "  fig = go.Figure()\n",
        "  fig.add_trace(go.Scatter(x=n_epochs, y=train_accuracy,\n",
        "                  mode='lines+markers',name='train_accuracy'))\n",
        "\n",
        "  fig.add_trace(go.Scatter(x=n_epochs, y=validation_accuracy,\n",
        "                  mode='lines+markers',name='validation_accuracy'))\n",
        "\n",
        "  fig.update_layout(title='Train_accuracy and Validation_accuracy',xaxis_title='number of epochs',yaxis_title='Accuracy')\n",
        "  fig.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X3NsyKfEVBNY",
        "colab_type": "text"
      },
      "source": [
        "## Model -1 \n",
        "Basic Architecture of Convnet Only **3 layers of ConvNets** (Without any BatchNormalization or any complex layers) **Without** **padding and MaxPooling layer and with default stride**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOfZmPZDVChB",
        "colab_type": "code",
        "outputId": "8d817c9e-c66f-4b6f-94e8-6fdec22df1f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "model_1 = Sequential()\n",
        "model_1.add(Conv2D(128,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   input_shape=input_shape))\n",
        "model_1.add(Conv2D(64,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "model_1.add(Conv2D(32,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "\n",
        "\n",
        "model_1.add(Dropout(0.25))\n",
        "model_1.add(Flatten())\n",
        "model_1.add(Dense(128,activation='relu'))\n",
        "model_1.add(Dropout(0.25))\n",
        "model_1.add(Dense(10,activation='softmax'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cNwpUSajVC2R",
        "colab_type": "code",
        "outputId": "40a43418-17e5-41fd-b1da-2ef318054105",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "model_1.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 26, 26, 128)       1280      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 24, 24, 64)        73792     \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 22, 22, 32)        18464     \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 22, 22, 32)        0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 15488)             0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 128)               1982592   \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                1290      \n",
            "=================================================================\n",
            "Total params: 2,077,418\n",
            "Trainable params: 2,077,418\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fk58zQNQVCzt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_1.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zPr7EuTWVCx_",
        "colab_type": "code",
        "outputId": "af9dc80c-a3b5-48c9-ad8c-62780a6b5153",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "history = model_1.fit(X_train,Y_train,batch_size=batch_size,epochs=epochs,validation_data=(X_test,Y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/20\n",
            "60000/60000 [==============================] - 560s 9ms/sample - loss: 0.1643 - acc: 0.9501 - val_loss: 0.0452 - val_acc: 0.9854\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 585s 10ms/sample - loss: 0.0522 - acc: 0.9840 - val_loss: 0.0299 - val_acc: 0.9893\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 606s 10ms/sample - loss: 0.0384 - acc: 0.9879 - val_loss: 0.0373 - val_acc: 0.9874\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 603s 10ms/sample - loss: 0.0296 - acc: 0.9905 - val_loss: 0.0292 - val_acc: 0.9900\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 592s 10ms/sample - loss: 0.0243 - acc: 0.9925 - val_loss: 0.0289 - val_acc: 0.9907\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 567s 9ms/sample - loss: 0.0184 - acc: 0.9939 - val_loss: 0.0297 - val_acc: 0.9910\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 589s 10ms/sample - loss: 0.0159 - acc: 0.9949 - val_loss: 0.0300 - val_acc: 0.9919\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 605s 10ms/sample - loss: 0.0144 - acc: 0.9950 - val_loss: 0.0319 - val_acc: 0.9917\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 602s 10ms/sample - loss: 0.0131 - acc: 0.9955 - val_loss: 0.0301 - val_acc: 0.9918\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 591s 10ms/sample - loss: 0.0124 - acc: 0.9957 - val_loss: 0.0388 - val_acc: 0.9908\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 604s 10ms/sample - loss: 0.0114 - acc: 0.9960 - val_loss: 0.0304 - val_acc: 0.9912\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 569s 9ms/sample - loss: 0.0082 - acc: 0.9972 - val_loss: 0.0375 - val_acc: 0.9910\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 565s 9ms/sample - loss: 0.0092 - acc: 0.9968 - val_loss: 0.0390 - val_acc: 0.9905\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 569s 9ms/sample - loss: 0.0074 - acc: 0.9974 - val_loss: 0.0340 - val_acc: 0.9913\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 593s 10ms/sample - loss: 0.0087 - acc: 0.9972 - val_loss: 0.0296 - val_acc: 0.9921\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 564s 9ms/sample - loss: 0.0078 - acc: 0.9972 - val_loss: 0.0378 - val_acc: 0.9913\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 575s 10ms/sample - loss: 0.0065 - acc: 0.9979 - val_loss: 0.0364 - val_acc: 0.9918\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 563s 9ms/sample - loss: 0.0073 - acc: 0.9975 - val_loss: 0.0345 - val_acc: 0.9917\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 562s 9ms/sample - loss: 0.0050 - acc: 0.9982 - val_loss: 0.0370 - val_acc: 0.9924\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 569s 9ms/sample - loss: 0.0046 - acc: 0.9984 - val_loss: 0.0398 - val_acc: 0.9914\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ITUicpncVCv-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "score = model_1.evaluate(X_test,Y_test,verbose=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47CDf3H4VCur",
        "colab_type": "code",
        "outputId": "ff426bb4-4763-4441-95a2-66590322e1d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(f\"Test score is {score[0]}\")\n",
        "print(f\"Test accuracy is {score[1]}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test score is 0.03980802101993977\n",
            "Test accuracy is 0.9914000034332275\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GL5LA2D6VCta",
        "colab_type": "code",
        "outputId": "8ac9acf3-76d4-408c-ad21-b67c82e82848",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "validation_loss = history.history['val_loss']\n",
        "train_loss = history.history['loss']\n",
        "train_accuracy = history.history['acc']\n",
        "validation_accuracy = history.history['val_acc']\n",
        "loss_plots(no_of_epochs,train_loss,validation_loss)\n",
        "accuracy_plots(no_of_epochs,train_accuracy,validation_accuracy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"3dcb0298-b792-4a19-b172-98fa100415b6\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"3dcb0298-b792-4a19-b172-98fa100415b6\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '3dcb0298-b792-4a19-b172-98fa100415b6',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.16425851961250107, 0.05219646644294262, 0.03842499006489913, 0.0296076394384106, 0.024263211162822942, 0.018424468804585436, 0.01594282167768106, 0.014373267471619572, 0.013148307581276943, 0.012376835269170503, 0.011398963342575978, 0.00816721135750413, 0.00921738991062933, 0.007423009580814202, 0.008692337511464332, 0.007789596355108855, 0.00652235038872653, 0.007302179019781518, 0.004970788883023973, 0.004583503549413096]}, {\"mode\": \"lines+markers\", \"name\": \"validation_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.045210549294948575, 0.029859181978181003, 0.03726996344812215, 0.029186177081614733, 0.02888230402581394, 0.029722921732068062, 0.03004372538998723, 0.03187774615208618, 0.0300825244813459, 0.03882605580912714, 0.03041285317353904, 0.03754706886087079, 0.03902329520656276, 0.034003177062235775, 0.02956508591566235, 0.03780497154953191, 0.03638932476585032, 0.034474707985669366, 0.036962730405712275, 0.03980801956728101]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_loss and Validation_loss\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Categorical Cross Entropy Loss\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('3dcb0298-b792-4a19-b172-98fa100415b6');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"2a667f32-e81e-4db8-ab6d-30a6b22d5694\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"2a667f32-e81e-4db8-ab6d-30a6b22d5694\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '2a667f32-e81e-4db8-ab6d-30a6b22d5694',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.950083315372467, 0.9840499758720398, 0.9879166483879089, 0.9904833436012268, 0.9925333261489868, 0.9939166903495789, 0.994949996471405, 0.9949666857719421, 0.9954833388328552, 0.9957000017166138, 0.9959999918937683, 0.9972166419029236, 0.9967666864395142, 0.9973833560943604, 0.9971833229064941, 0.9971500039100647, 0.9979166388511658, 0.9975000023841858, 0.9982166886329651, 0.9984166622161865]}, {\"mode\": \"lines+markers\", \"name\": \"validation_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.9854000210762024, 0.989300012588501, 0.9873999953269958, 0.9900000095367432, 0.9907000064849854, 0.9909999966621399, 0.9919000267982483, 0.9916999936103821, 0.9918000102043152, 0.9908000230789185, 0.9911999702453613, 0.9909999966621399, 0.9904999732971191, 0.9912999868392944, 0.9921000003814697, 0.9912999868392944, 0.9918000102043152, 0.9916999936103821, 0.9923999905586243, 0.9914000034332275]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_accuracy and Validation_accuracy\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Accuracy\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('2a667f32-e81e-4db8-ab6d-30a6b22d5694');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdJmqXrDVCri",
        "colab_type": "text"
      },
      "source": [
        "**Observation**\n",
        "*   There is slight overfit in the model (which can be interpreted from both the plots)\n",
        "*   ideal epochs might be considered as 2 or 3 in this architecture because when only at that stage both train loss and test loss are close\n",
        "*   This is very basic architecture without batch normalization and max pooling and padding. Let's see further cases.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dd-g0xclVCmv",
        "colab_type": "text"
      },
      "source": [
        "## Model - 2 \n",
        "Slightly complex Architecture of Convnet Only **5 layers of ConvNets** (With BatchNormalization and MaxPooling and slight complex layers) **Without** **padding and with default stride**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Frr-Sry7HEDL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.layers import BatchNormalization"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MxRqSpFSF-eu",
        "outputId": "208be5f4-5d6c-4735-b08d-8a7ba677495a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "model_2 = Sequential()\n",
        "model_2.add(Conv2D(256,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   input_shape=input_shape))\n",
        "model_2.add(Conv2D(128,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "model_2.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model_2.add(Conv2D(64,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "model_2.add(Conv2D(32,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "model_2.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model_2.add(Conv2D(16,kernel_size=(3,3),\n",
        "                   activation='relu'))\n",
        "\n",
        "model_2.add(Flatten())\n",
        "model_2.add(Dense(128,activation='relu'))\n",
        "model_2.add(BatchNormalization())\n",
        "model_2.add(Dropout(0.25))\n",
        "model_2.add(Dense(64,activation='relu'))\n",
        "model_2.add(BatchNormalization())\n",
        "model_2.add(Dropout(0.25))\n",
        "model_2.add(Dense(10,activation='softmax'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "dfad63b9-c415-45ae-8de6-066a5b4dd705",
        "id": "rhH-BUXQF-e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 663
        }
      },
      "source": [
        "model_2.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 26, 26, 256)       2560      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 24, 24, 128)       295040    \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 12, 12, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 10, 10, 64)        73792     \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 8, 8, 32)          18464     \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 4, 4, 32)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 2, 2, 16)          4624      \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 128)               8320      \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 128)               512       \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 64)                8256      \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 64)                256       \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 10)                650       \n",
            "=================================================================\n",
            "Total params: 412,474\n",
            "Trainable params: 412,090\n",
            "Non-trainable params: 384\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sVxUl4ZJF-fD",
        "colab": {}
      },
      "source": [
        "model_2.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "25d3affb-15ef-4f7a-80ec-32410af10b8b",
        "id": "LNO7M4NKF-fL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "history = model_2.fit(X_train,Y_train,batch_size=batch_size,epochs=epochs,validation_data=(X_test,Y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/20\n",
            "60000/60000 [==============================] - 38s 631us/sample - loss: 0.2730 - acc: 0.9190 - val_loss: 0.0787 - val_acc: 0.9785\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 31s 514us/sample - loss: 0.0755 - acc: 0.9787 - val_loss: 0.0350 - val_acc: 0.9892\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 31s 517us/sample - loss: 0.0551 - acc: 0.9838 - val_loss: 0.0464 - val_acc: 0.9860\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 31s 520us/sample - loss: 0.0447 - acc: 0.9865 - val_loss: 0.0258 - val_acc: 0.9923\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 31s 510us/sample - loss: 0.0377 - acc: 0.9883 - val_loss: 0.0279 - val_acc: 0.9909\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 31s 517us/sample - loss: 0.0331 - acc: 0.9901 - val_loss: 0.0806 - val_acc: 0.9769\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 31s 521us/sample - loss: 0.0291 - acc: 0.9909 - val_loss: 0.0283 - val_acc: 0.9918\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 31s 517us/sample - loss: 0.0259 - acc: 0.9917 - val_loss: 0.0347 - val_acc: 0.9893\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 30s 507us/sample - loss: 0.0225 - acc: 0.9936 - val_loss: 0.0372 - val_acc: 0.9885\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 31s 510us/sample - loss: 0.0214 - acc: 0.9930 - val_loss: 0.0307 - val_acc: 0.9916\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 31s 511us/sample - loss: 0.0200 - acc: 0.9936 - val_loss: 0.0295 - val_acc: 0.9916\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 31s 510us/sample - loss: 0.0160 - acc: 0.9950 - val_loss: 0.0238 - val_acc: 0.9936\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 31s 514us/sample - loss: 0.0139 - acc: 0.9960 - val_loss: 0.0425 - val_acc: 0.9892\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 31s 521us/sample - loss: 0.0139 - acc: 0.9954 - val_loss: 0.0336 - val_acc: 0.9904\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 31s 522us/sample - loss: 0.0121 - acc: 0.9959 - val_loss: 0.0330 - val_acc: 0.9907\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 30s 508us/sample - loss: 0.0120 - acc: 0.9962 - val_loss: 0.0366 - val_acc: 0.9897\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 30s 508us/sample - loss: 0.0102 - acc: 0.9969 - val_loss: 0.0236 - val_acc: 0.9941\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 31s 509us/sample - loss: 0.0109 - acc: 0.9963 - val_loss: 0.0269 - val_acc: 0.9917\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 31s 508us/sample - loss: 0.0080 - acc: 0.9974 - val_loss: 0.0252 - val_acc: 0.9929\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 31s 511us/sample - loss: 0.0083 - acc: 0.9974 - val_loss: 0.0309 - val_acc: 0.9925\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ed5TZJs0F-fR",
        "colab": {}
      },
      "source": [
        "score = model_2.evaluate(X_test,Y_test,verbose=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "twSYmazXF-fW",
        "outputId": "54a62b98-b621-4e3c-eb53-d20f5d03e7d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(f\"Test score is {score[0]}\")\n",
        "print(f\"Test accuracy is {score[1]}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test score is 0.030873713541904362\n",
            "Test accuracy is 0.9925000071525574\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zThufS85F-fb",
        "outputId": "e0a403a6-f826-414d-a378-1117ab0e706c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "validation_loss = history.history['val_loss']\n",
        "train_loss = history.history['loss']\n",
        "train_accuracy = history.history['acc']\n",
        "validation_accuracy = history.history['val_acc']\n",
        "loss_plots(no_of_epochs,train_loss,validation_loss)\n",
        "accuracy_plots(no_of_epochs,train_accuracy,validation_accuracy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"5d1e66fd-5961-426c-9956-c13f058132d8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"5d1e66fd-5961-426c-9956-c13f058132d8\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '5d1e66fd-5961-426c-9956-c13f058132d8',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.27300841588974, 0.07549657495617866, 0.055129665383696555, 0.04469278329014778, 0.03773167960345745, 0.033053625483810904, 0.029106020800272622, 0.025863287906224528, 0.0225037971979628, 0.02137118894333641, 0.01995456544601669, 0.01604540536087006, 0.013862054132297635, 0.013890503024496138, 0.012057775558996946, 0.012029709929196785, 0.010202192658061783, 0.01090837204537044, 0.00803702695000296, 0.008265485961929274]}, {\"mode\": \"lines+markers\", \"name\": \"validation_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.07865545092225075, 0.034978141866019, 0.046420230238325894, 0.0257700847029686, 0.0279442772179842, 0.08056756167709828, 0.028341435202304272, 0.03471245311592938, 0.03723794624060393, 0.030698640883713962, 0.029455441754008643, 0.023822332480549813, 0.0424628508443886, 0.033640170657634735, 0.03303050983436406, 0.03657088244725019, 0.023570305531984196, 0.026913121510576458, 0.025161118942495522, 0.03087371091619134]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_loss and Validation_loss\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Categorical Cross Entropy Loss\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('5d1e66fd-5961-426c-9956-c13f058132d8');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"9705bca9-6147-4013-ac84-c68c42b52cd3\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"9705bca9-6147-4013-ac84-c68c42b52cd3\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '9705bca9-6147-4013-ac84-c68c42b52cd3',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.9189666509628296, 0.9787499904632568, 0.9837999939918518, 0.9865166544914246, 0.9882833361625671, 0.9900833368301392, 0.9908999800682068, 0.9916666746139526, 0.9935666918754578, 0.9929999709129333, 0.9935833215713501, 0.9949666857719421, 0.9959666728973389, 0.9954000115394592, 0.9959333539009094, 0.9962499737739563, 0.9968666434288025, 0.9962666630744934, 0.9973666667938232, 0.9974333047866821]}, {\"mode\": \"lines+markers\", \"name\": \"validation_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.9785000085830688, 0.9891999959945679, 0.9860000014305115, 0.9922999739646912, 0.9908999800682068, 0.9768999814987183, 0.9918000102043152, 0.989300012588501, 0.9884999990463257, 0.991599977016449, 0.991599977016449, 0.9936000108718872, 0.9891999959945679, 0.9904000163078308, 0.9907000064849854, 0.9897000193595886, 0.9940999746322632, 0.9916999936103821, 0.992900013923645, 0.9925000071525574]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_accuracy and Validation_accuracy\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Accuracy\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('9705bca9-6147-4013-ac84-c68c42b52cd3');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pk3lV-HeVCdU",
        "colab_type": "text"
      },
      "source": [
        "**Observation**\n",
        "*   Even in this model, there is overfit (which can be interpreted from both the plots)\n",
        "*   As we have seen in the example of Resnet as no.of.layers are increasing the model's performance is getting worse, the better approach would be to use hyperparameter tuning to find accurate number of epochs, moreover there are architecture and kernel size might also be the issue (which are again hyperparameters).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "v17bEgB7DasU"
      },
      "source": [
        "## Model - 3\n",
        "Complex Architecture of Convnet Only **6 layers of ConvNets** (With BatchNormalization and MaxPooling and **With** **padding and default stride**)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kULpaW-wDasc",
        "colab": {}
      },
      "source": [
        "model_3 = Sequential()\n",
        "model_3.add(Conv2D(512,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same',\n",
        "                   input_shape=input_shape))\n",
        "model_3.add(Conv2D(128,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same'))\n",
        "model_3.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model_3.add(Conv2D(64,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same'))\n",
        "model_3.add(Conv2D(32,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same'))\n",
        "model_3.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model_3.add(Conv2D(16,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same'))\n",
        "model_3.add(Conv2D(32,kernel_size=(3,3),\n",
        "                   activation='relu',\n",
        "                   padding='same'))\n",
        "model_3.add(MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model_3.add(Flatten())\n",
        "model_3.add(Dense(256,activation='relu'))\n",
        "model_3.add(BatchNormalization())\n",
        "model_3.add(Dropout(0.20))\n",
        "model_3.add(Dense(128,activation='relu'))\n",
        "model_3.add(BatchNormalization())\n",
        "model_3.add(Dropout(0.20))\n",
        "model_3.add(Dense(64,activation='relu'))\n",
        "model_3.add(BatchNormalization())\n",
        "model_3.add(Dropout(0.20))\n",
        "model_3.add(Dense(10,activation='softmax'))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "52eb16a8-c4a5-40fd-f026-166d52db164b",
        "id": "N62AitCNDasg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 833
        }
      },
      "source": [
        "model_3.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_5 (Conv2D)            (None, 28, 28, 512)       5120      \n",
            "_________________________________________________________________\n",
            "conv2d_6 (Conv2D)            (None, 28, 28, 128)       589952    \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 14, 14, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_7 (Conv2D)            (None, 14, 14, 64)        73792     \n",
            "_________________________________________________________________\n",
            "conv2d_8 (Conv2D)            (None, 14, 14, 32)        18464     \n",
            "_________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2 (None, 7, 7, 32)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_9 (Conv2D)            (None, 7, 7, 16)          4624      \n",
            "_________________________________________________________________\n",
            "conv2d_10 (Conv2D)           (None, 7, 7, 32)          4640      \n",
            "_________________________________________________________________\n",
            "max_pooling2d_4 (MaxPooling2 (None, 3, 3, 32)          0         \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 288)               0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 256)               73984     \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 256)               1024      \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 128)               32896     \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 128)               512       \n",
            "_________________________________________________________________\n",
            "dropout_3 (Dropout)          (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 64)                8256      \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 64)                256       \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 10)                650       \n",
            "=================================================================\n",
            "Total params: 814,170\n",
            "Trainable params: 813,274\n",
            "Non-trainable params: 896\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6Mkn_od2Dasm",
        "colab": {}
      },
      "source": [
        "model_3.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "098c7acd-e9d1-49d5-8590-37c606d7724a",
        "id": "oZ-XVaLzDast",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "history = model_3.fit(X_train,Y_train,batch_size=batch_size,epochs=epochs,validation_data=(X_test,Y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/20\n",
            "60000/60000 [==============================] - 59s 984us/sample - loss: 0.2432 - acc: 0.9284 - val_loss: 0.1052 - val_acc: 0.9673\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 56s 930us/sample - loss: 0.0677 - acc: 0.9806 - val_loss: 0.0943 - val_acc: 0.9703\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 56s 928us/sample - loss: 0.0503 - acc: 0.9854 - val_loss: 0.1281 - val_acc: 0.9622\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 56s 928us/sample - loss: 0.0431 - acc: 0.9873 - val_loss: 0.0380 - val_acc: 0.9873\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 56s 927us/sample - loss: 0.0360 - acc: 0.9893 - val_loss: 0.0680 - val_acc: 0.9801\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 56s 928us/sample - loss: 0.0326 - acc: 0.9905 - val_loss: 0.0519 - val_acc: 0.9847\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 56s 926us/sample - loss: 0.0279 - acc: 0.9916 - val_loss: 0.0368 - val_acc: 0.9891\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 56s 926us/sample - loss: 0.0261 - acc: 0.9917 - val_loss: 0.0322 - val_acc: 0.9904\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 56s 925us/sample - loss: 0.0233 - acc: 0.9927 - val_loss: 0.1226 - val_acc: 0.9628\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 55s 924us/sample - loss: 0.0231 - acc: 0.9928 - val_loss: 0.0244 - val_acc: 0.9924\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 56s 934us/sample - loss: 0.0202 - acc: 0.9934 - val_loss: 0.0376 - val_acc: 0.9876\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 56s 927us/sample - loss: 0.0177 - acc: 0.9950 - val_loss: 0.0257 - val_acc: 0.9922\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 56s 927us/sample - loss: 0.0192 - acc: 0.9942 - val_loss: 0.0625 - val_acc: 0.9839\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 56s 926us/sample - loss: 0.0168 - acc: 0.9949 - val_loss: 0.0388 - val_acc: 0.9900\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 56s 929us/sample - loss: 0.0163 - acc: 0.9948 - val_loss: 0.0239 - val_acc: 0.9928\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 56s 930us/sample - loss: 0.0131 - acc: 0.9960 - val_loss: 0.0494 - val_acc: 0.9851\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 56s 927us/sample - loss: 0.0139 - acc: 0.9955 - val_loss: 0.0181 - val_acc: 0.9945\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 56s 932us/sample - loss: 0.0113 - acc: 0.9964 - val_loss: 0.0321 - val_acc: 0.9911\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 56s 939us/sample - loss: 0.0123 - acc: 0.9961 - val_loss: 0.0254 - val_acc: 0.9930\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 56s 930us/sample - loss: 0.0101 - acc: 0.9966 - val_loss: 0.0254 - val_acc: 0.9937\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Hs7xhhnBDasx",
        "colab": {}
      },
      "source": [
        "score = model_3.evaluate(X_test,Y_test,verbose=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "11104640-7d0a-4cd5-daa2-d609ce497f96",
        "id": "ZaDu9efBDas4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(f\"Test score is {score[0]}\")\n",
        "print(f\"Test accuracy is {score[1]}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test score is 0.02541501125908253\n",
            "Test accuracy is 0.9937000274658203\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "d7aa6e76-a362-42f8-ba35-392e04299441",
        "id": "HUZMRg8lDas9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "validation_loss = history.history['val_loss']\n",
        "train_loss = history.history['loss']\n",
        "train_accuracy = history.history['acc']\n",
        "validation_accuracy = history.history['val_acc']\n",
        "loss_plots(no_of_epochs,train_loss,validation_loss)\n",
        "accuracy_plots(no_of_epochs,train_accuracy,validation_accuracy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"ae2a50b8-24c7-4731-ac0a-19226de2e414\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"ae2a50b8-24c7-4731-ac0a-19226de2e414\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        'ae2a50b8-24c7-4731-ac0a-19226de2e414',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.24318152308066687, 0.06773539410233498, 0.05030909374058247, 0.04305568776826064, 0.036016164992252984, 0.03255235765998562, 0.027908972262839477, 0.026087969968964658, 0.023329975102096796, 0.023116740530977647, 0.020192838323364656, 0.017709984784945846, 0.01920231886940698, 0.016768811963188152, 0.016300277959865828, 0.013134443517401815, 0.013918123984088501, 0.011292199414037168, 0.01231576178331549, 0.010070273332297801]}, {\"mode\": \"lines+markers\", \"name\": \"validation_loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.10522064654827118, 0.09431660635471344, 0.12812219406720252, 0.03800056308060885, 0.06796809511072933, 0.05187750984579325, 0.036825242096185686, 0.032199592409282926, 0.12257302726507187, 0.024376010714087168, 0.037550944589450956, 0.025736500584799798, 0.06251004088548943, 0.038838538564369085, 0.023866440534591676, 0.04941500393748283, 0.018075283964723347, 0.032073664098849985, 0.025394667126238345, 0.025415009563149942]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_loss and Validation_loss\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Categorical Cross Entropy Loss\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('ae2a50b8-24c7-4731-ac0a-19226de2e414');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"26c95ba2-ecb5-4eaf-bc81-5cb9a8f7441f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"26c95ba2-ecb5-4eaf-bc81-5cb9a8f7441f\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '26c95ba2-ecb5-4eaf-bc81-5cb9a8f7441f',\n",
              "                        [{\"mode\": \"lines+markers\", \"name\": \"train_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.928433358669281, 0.9806166887283325, 0.9853833317756653, 0.987333357334137, 0.989300012588501, 0.9904833436012268, 0.9916166663169861, 0.9916666746139526, 0.9926999807357788, 0.9927666783332825, 0.9934333562850952, 0.9949833154678345, 0.994183361530304, 0.994866669178009, 0.9947500228881836, 0.9959999918937683, 0.9955499768257141, 0.9964166879653931, 0.9960833191871643, 0.9965833425521851]}, {\"mode\": \"lines+markers\", \"name\": \"validation_accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], \"y\": [0.9672999978065491, 0.970300018787384, 0.9621999859809875, 0.9872999787330627, 0.9800999760627747, 0.9847000241279602, 0.9890999794006348, 0.9904000163078308, 0.9628000259399414, 0.9923999905586243, 0.9876000285148621, 0.9922000169754028, 0.9839000105857849, 0.9900000095367432, 0.9927999973297119, 0.9850999712944031, 0.9944999814033508, 0.991100013256073, 0.9929999709129333, 0.9937000274658203]}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Train_accuracy and Validation_accuracy\"}, \"xaxis\": {\"title\": {\"text\": \"number of epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Accuracy\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('26c95ba2-ecb5-4eaf-bc81-5cb9a8f7441f');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u9ZQzkKqDRZJ",
        "colab_type": "text"
      },
      "source": [
        "**Final Observation**\n",
        "\n",
        "Among all the architectures only basic architecture which we implemented as **model_1 has performed better than other complex architectures**. This might be because of various hyperparameters like no.of.layers , kernel size and padding, stride, number of layers and many more."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKITvpMrDQz1",
        "colab_type": "text"
      },
      "source": [
        "# ---------------------------------------------------------------------------------------------------------------------------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XuydltoqVCaj",
        "colab_type": "text"
      },
      "source": [
        "**IMPORTANT NOTES**\n",
        "\n",
        "\n",
        "1.   **Question :** If i have an image, i scrambled(reorder the pixels for security of image)\n",
        "Then trained the CNN on it,\n",
        "Will CNN learn same things as if trained it on original image.\n",
        "       \n",
        "       **ANSWER by Applied AI Course** \n",
        "Very good question. In some forms of scrambling like moving part of images like this, CNNs can learn the classes of the orignial images well. But, if you completely randomize the pixels, it would be significantly harder. Geoff Hinton designed a new type of models called capsule networks to work around this limitation of CNNs that they do not respect the relative positions and geometry of objects in an image.\n",
        "\n",
        "\n",
        "\n",
        "2. **Question:** Why-is-max-pooling-necessary-in-convolutional-neural-networks? Most common convolutional neural networks contains pooling layers to reduce the dimensions of output features. Why couldn't I achieve the same thing by simply increase the stride of the convolutional layer? What makes the pooling layer necessary? \n",
        " \n",
        "  **Answer:**\n",
        "You can indeed do that, see [Striving for Simplicity](https://arxiv.org/abs/1412.6806): The All Convolutional Net. Pooling gives you some amount of translation invariance, which may or may not be helpful. Also, pooling is faster to compute than convolutions. Still, you can always try replacing pooling by convolution with stride and see what works better.\n",
        " Some current works use average pooling ( [Wide Residual Networks](https://arxiv.org/abs/1605.07146), [DenseNets](https://arxiv.org/abs/1608.06993)), others use convolution with stride ( [DelugeNets](https://arxiv.org/abs/1611.05552))\n",
        "  https://stats.stackexchange.com/questions/288261/why-is-max-pooling-necessary-in-convolutional-neural-networks\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWtCYUKQzTu5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}