IST597_HW4_MLP.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ZhaoQii/IST597/blob/main/IST597_HW4_MLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "PXlJ9Shye-8f"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "#np.random.seed(1234)\n",
        "#tf.random.set_seed(1234)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vJi6QDIgfLDV",
        "outputId": "271338b6-a859-4719-fefe-6cf94b9ded41"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "size of training set is 50000 samples\n",
            "every train example is 28 by 28\n",
            "size of validation set is 10000 samples\n",
            "every validation example is 28 by 28\n",
            "size of training set is 50000 samples\n",
            "every train example has 784 features\n",
            "size of validation set is 10000 samples\n",
            "every validation example has 784 features\n"
          ]
        }
      ],
      "source": [
        "(X_train, y_train), (X_test, y_test) = keras.datasets.fashion_mnist.load_data() # Load MNIST or FMNIST\n",
        "assert X_train.shape == (60000, 28, 28)\n",
        "assert X_test.shape == (10000, 28, 28)\n",
        "assert y_train.shape == (60000,)\n",
        "assert y_test.shape == (10000,)\n",
        "\n",
        "# Split train dataset into train and validation\n",
        "X_val = X_train[50000:60000]\n",
        "X_train = X_train[0:50000]\n",
        "y_val = y_train[50000:60000]\n",
        "y_train = y_train[0:50000]\n",
        "\n",
        "print(\"size of training set is\", str(X_train.shape[0]), \"samples\")\n",
        "print(\"every train example is\", str(X_train.shape[1]), \"by\", str(X_train.shape[2]))\n",
        "\n",
        "print(\"size of validation set is\", str(X_val.shape[0]), \"samples\")\n",
        "print(\"every validation example is\", str(X_val.shape[1]), \"by\", str(X_val.shape[2]))\n",
        "\n",
        "X_train = X_train.reshape(50000, 28*28)\n",
        "X_val = X_val.reshape(10000, 28*28)\n",
        "X_test = X_test.reshape(10000, 28*28)\n",
        "\n",
        "print(\"size of training set is\", str(X_train.shape[0]), \"samples\")\n",
        "print(\"every train example has\", str(X_train.shape[1]), \"features\")\n",
        "\n",
        "print(\"size of validation set is\", str(X_val.shape[0]), \"samples\")\n",
        "print(\"every validation example has\", str(X_val.shape[1]), \"features\")\n",
        "\n",
        "# Split dataset into batches\n",
        "#train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).batch(16)\n",
        "#test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tAG_B8DafN2W",
        "outputId": "381dfbb8-a32a-4be1-da41-844fea0aaea6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "#Normalize Data\n",
        "\n",
        "X_train = X_train/255\n",
        "X_val = X_val/255\n",
        "X_test = X_test/255\n",
        "# X_train[0]\n",
        "np.max(X_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WRggUB0PfPnG",
        "outputId": "18817b18-0b54-4d0f-e2e0-4d7382ab3627"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor([10000    10], shape=(2,), dtype=int32)\n"
          ]
        }
      ],
      "source": [
        "size_input = X_train.shape[1]\n",
        "size_hidden1 = 128\n",
        "size_hidden2 = 128\n",
        "size_hidden3 = 128\n",
        "size_output = 10\n",
        "batch_size = 64\n",
        "\n",
        "number_of_train_examples = X_train.shape[0]\n",
        "number_of_test_examples = X_test.shape[0]\n",
        "\n",
        "y_train = tf.keras.utils.to_categorical(y_train, num_classes=10) # Other function is tf.one_hot(y_train,depth=10)\n",
        "y_val = tf.keras.utils.to_categorical(y_val, num_classes=10)\n",
        "y_test = tf.keras.utils.to_categorical(y_test, num_classes=10)\n",
        "print(tf.shape(y_val))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "CL6nQDObudYM"
      },
      "outputs": [],
      "source": [
        "# Define class to build mlp model\n",
        "class MLP_NB(object):\n",
        " def __init__(self, size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type, epsilon = 1e-3, device = None):\n",
        "    \"\"\"\n",
        "    size_input: int, size of input layer\n",
        "    size_hidden1: int, size of the 1st hidden layer\n",
        "    size_hidden2: int, size of the 2nd hidden layer\n",
        "    size_output: int, size of output layer\n",
        "    batch_size\n",
        "    BN_type: 'before' for BN after pre-activation, 'after' for BN after post-activation\n",
        "    device: str or None, either 'cpu' or 'gpu' or None. If None, the device to be used will be decided automatically during Eager Execution\n",
        "    \"\"\"\n",
        "    self.size_input, self.size_hidden1, self.size_hidden2, self.size_hidden3, self.size_output, self.BN_type, self.epsilon, self.device =\\\n",
        "    size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type, epsilon, device\n",
        "    \n",
        "    # Initialize weights between input mapping and a layer g(f(x)) = layer\n",
        "    self.W1 = tf.Variable(tf.random.normal([self.size_input, self.size_hidden1],stddev=0.1)) # Xavier(Fan-in fan-out) and Orthogonal\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b1 = tf.Variable(tf.zeros([1, self.size_hidden1])) # 0 or constant(0.01)\n",
        "    # Initialize biases for BN\n",
        "    self.scale1 = tf.Variable(tf.ones([self.size_hidden1]))\n",
        "    self.bias1 = tf.Variable(tf.zeros([self.size_hidden1]))\n",
        "    \n",
        "    # Initialize weights between input layer and 1st hidden layer\n",
        "    self.W2 = tf.Variable(tf.random.normal([self.size_hidden1, self.size_hidden2],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b2 = tf.Variable(tf.zeros([1, self.size_hidden2]))\n",
        "    # Initialize biases for BN\n",
        "    self.scale2 = tf.Variable(tf.ones([self.size_hidden2]))\n",
        "    self.bias2 = tf.Variable(tf.zeros([self.size_hidden2]))\n",
        "    \n",
        "    # Initialize weights between 1st hidden layer and 2nd hidden layer\n",
        "    self.W3 = tf.Variable(tf.random.normal([self.size_hidden2, self.size_hidden3],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b3 = tf.Variable(tf.zeros([1, self.size_hidden3]))\n",
        "    # Initialize biases for BN\n",
        "    self.scale3 = tf.Variable(tf.ones([self.size_hidden3]))\n",
        "    self.bias3 = tf.Variable(tf.zeros([self.size_hidden3]))\n",
        "\n",
        "     # Initialize weights between 2nd hidden layer and output layer\n",
        "    self.W4 = tf.Variable(tf.random.normal([self.size_hidden3, self.size_output],stddev=0.1))\n",
        "    # Initialize biases for output layer\n",
        "    self.b4 = tf.Variable(tf.zeros([1, self.size_output]))\n",
        "    # Initialize biases for BN\n",
        "    self.scale4 = tf.Variable(tf.ones([self.size_output]))\n",
        "    self.bias4 = tf.Variable(tf.zeros([self.size_output]))\n",
        "    \n",
        "    # Define variables to be updated during backpropagation\n",
        "    if self.BN_type == 'before':\n",
        "      # for before, then no need to do pre BN bias since it will be deleted by averaging anyway, so don't update b's for this case\n",
        "      # however, should include b4 if the final output layer does not involve the BN\n",
        "      self.variables = [self.W1, self.W2, self.W3, self.W4, self.b4,\n",
        "                        self.scale1, self.scale2, self.scale3, #, self.scale4,\n",
        "                        self.bias1, self.bias2, self.bias3]#, self.bias4]\n",
        "    else:\n",
        "      self.variables = [self.W1, self.W2, self.W3, self.W4, self.b1, self.b2, self.b3, self.b4,\n",
        "                        self.scale1, self.scale2, self.scale3, #, self.scale4,\n",
        "                        self.bias1, self.bias2, self.bias3]#, self.bias4]\n",
        "\n",
        " def forward(self, X, training, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None):\n",
        "    \"\"\"\n",
        "    forward pass\n",
        "    X: Tensor, inputs\n",
        "    \"\"\"\n",
        "    if self.device is not None:\n",
        "      with tf.device('gpu:0' if self.device=='gpu' else 'cpu'):\n",
        "        if self.BN_type == 'before':\n",
        "          self.y = self.compute_output_before(X, training, mean1, var1, mean2, var2, mean3, var3)\n",
        "        else:\n",
        "          self.y = self.compute_output_after(X, training, mean1, var1, mean2, var2, mean3, var3)\n",
        "    else:\n",
        "      if self.BN_type == 'before':\n",
        "        self.y = self.compute_output_before(X, training, mean1, var1, mean2, var2, mean3, var3)\n",
        "      else:\n",
        "        self.y = self.compute_output_after(X, training, mean1, var1, mean2, var2, mean3, var3)\n",
        "      \n",
        "    return self.y\n",
        "\n",
        " def loss(self, y_pred, y_true):\n",
        "    '''\n",
        "    y_pred - Tensor of shape (batch_size, size_output)\n",
        "    y_true - Tensor of shape (batch_size, size_output)\n",
        "    '''\n",
        "    #y_true_tf = tf.cast(tf.reshape(y_true, (-1, self.size_output)), dtype=tf.float32)\n",
        "    y_true_tf = tf.cast(y_true, dtype=tf.float32)\n",
        "    y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
        "    loss_x = cce(y_true_tf, y_pred_tf)\n",
        "    # Use keras or tf_softmax, both should work for any given model\n",
        "    #loss_x = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_pred_tf, labels=y_true_tf))\n",
        "    \n",
        "    return loss_x\n",
        "\n",
        " def backward(self, X_train, y_train, opti, training, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None):\n",
        "    \"\"\"\n",
        "    backward pass\n",
        "    \"\"\"\n",
        "    optimizer = opti\n",
        "    \n",
        "    with tf.GradientTape() as tape:\n",
        "        \n",
        "      predicted = self.forward(X_train, training, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None)\n",
        "      current_loss = self.loss(predicted, y_train)\n",
        "        \n",
        "    grads = tape.gradient(current_loss, self.variables)\n",
        "    optimizer.apply_gradients(zip(grads, self.variables))\n",
        "  \n",
        " def layer_mean_var(self, all_X):\n",
        "    all_X_tf = tf.cast(all_X, dtype = tf.float32)\n",
        "    if self.BN_type == 'before':\n",
        "      z1_all = tf.matmul(all_X_tf, self.W1) + self.b1\n",
        "      mean1, var1 = tf.nn.moments(z1_all, [0])\n",
        "      z1_BN_all = (z1_all - mean1) / tf.sqrt(var1 + self.epsilon)\n",
        "      BN1_all = self.scale1 * z1_BN_all + self.bias1\n",
        "      h1_all = tf.nn.relu(BN1_all)\n",
        "      \n",
        "      z2_all = tf.matmul(h1_all, self.W2) + self.b2\n",
        "      mean2, var2 = tf.nn.moments(z2_all, [0])\n",
        "      z2_BN_all = (z2_all - mean2) / tf.sqrt(var2 + self.epsilon)\n",
        "      BN2_all = self.scale2 * z2_BN_all + self.bias2\n",
        "      h2_all = tf.nn.relu(BN2_all)\n",
        "      \n",
        "      z3_all = tf.matmul(h2_all, self.W3) + self.b3\n",
        "      mean3, var3 = tf.nn.moments(z3_all, [0])\n",
        "    else:\n",
        "      z1_all = tf.matmul(all_X_tf, self.W1) + self.b1\n",
        "      z1_h1_all = tf.nn.relu(z1_all)\n",
        "      mean1, var1 = tf.nn.moments(z1_h1_all, [0])\n",
        "      z1_BN_all = (z1_h1_all - mean1) / tf.sqrt(var1 + self.epsilon)\n",
        "      h1_all = self.scale1 * z1_BN_all + self.bias1\n",
        "\n",
        "      z2_all = tf.matmul(h1_all, self.W2) + self.b2\n",
        "      z2_h2_all = tf.nn.relu(z2_all)\n",
        "      mean2, var2 = tf.nn.moments(z2_h2_all, [0])\n",
        "      z2_BN_all = (z2_h2_all - mean2) / tf.sqrt(var2 + self.epsilon)\n",
        "      h2_all = self.scale1 * z2_BN_all + self.bias2\n",
        "\n",
        "      z3_all = tf.matmul(h2_all, self.W3) + self.b3\n",
        "      z3_h3_all = tf.nn.relu(z3_all)\n",
        "      mean3, var3 = tf.nn.moments(z3_h3_all, [0])\n",
        "    \n",
        "    return(mean1, var1, mean2, var2, mean3, var3)\n",
        "\n",
        " def compute_output_before(self, X, training, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None):\n",
        "    \"\"\"\n",
        "    Custom method to obtain output tensor during forward pass\n",
        "    \"\"\"\n",
        "    # Cast X to float32\n",
        "    X_tf = tf.cast(X, dtype=tf.float32)\n",
        "    #X_tf = X\n",
        "\n",
        "    # Compute values in hidden layers\n",
        "    if training == True:\n",
        "      z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "      batch_mean1, batch_var1 = tf.nn.moments(z1, [0])\n",
        "      z1_BN = (z1 - batch_mean1) / tf.sqrt(batch_var1 + self.epsilon)\n",
        "      BN1 = self.scale1 * z1_BN + self.bias1\n",
        "      h1 = tf.nn.relu(BN1)\n",
        "      \n",
        "      z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "      batch_mean2, batch_var2 = tf.nn.moments(z2, [0])\n",
        "      z2_BN = (z2 - batch_mean2) / tf.sqrt(batch_var2 + self.epsilon)\n",
        "      BN2 = self.scale2 * z2_BN + self.bias2\n",
        "      h2 = tf.nn.relu(BN2)\n",
        "      \n",
        "      z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "      batch_mean3, batch_var3 = tf.nn.moments(z3, [0])\n",
        "      z3_BN = (z3 - batch_mean3) / tf.sqrt(batch_var3 + self.epsilon)\n",
        "      BN3 = self.scale3 * z3_BN + self.bias3\n",
        "      h3 = tf.nn.relu(BN3)\n",
        "\n",
        "    else:\n",
        "      z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "      z1_BN = (z1 - mean1) / tf.sqrt(var1 + self.epsilon)\n",
        "      BN1 = self.scale1 * z1_BN + self.bias1\n",
        "      h1 = tf.nn.relu(BN1)\n",
        "      \n",
        "      z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "      z2_BN = (z2 - mean2) / tf.sqrt(var2 + self.epsilon)\n",
        "      BN2 = self.scale2 * z2_BN + self.bias2\n",
        "      h2 = tf.nn.relu(BN2)\n",
        "      \n",
        "      z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "      z3_BN = (z3 - mean3) / tf.sqrt(var3 + self.epsilon)\n",
        "      BN3 = self.scale3 * z3_BN + self.bias3\n",
        "      h3 = tf.nn.relu(BN3)\n",
        "\n",
        "    # Compute output\n",
        "    output = tf.matmul(h3, self.W4) + self.b4\n",
        "\n",
        "    # as no BN for output layer if BN_type == 'after'\n",
        "    #batch_mean4, batch_var4 = tf.nn.moments(output, [0])\n",
        "    #z4_BN = (output - batch_mean4) / tf.sqrt(batch_var4 + self.epsilon)\n",
        "    #output = self.scale4 * z4_BN + self.bias4\n",
        "    \n",
        "    #Now consider two things , First look at inbuild loss functions if they work with softmax or not and then change this \n",
        "    # Second add tf.Softmax(output) and then return this variable\n",
        "    return (output)\n",
        "\n",
        " def compute_output_after(self, X, training, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None):\n",
        "    \"\"\"\n",
        "    Custom method to obtain output tensor during forward pass\n",
        "    \"\"\"\n",
        "    # Cast X to float32\n",
        "    X_tf = tf.cast(X, dtype=tf.float32)\n",
        "    #X_tf = X\n",
        "    # Compute values in hidden layers\n",
        "\n",
        "    if training == True:\n",
        "    \n",
        "      # Compute values in hidden layers\n",
        "      z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "      z1_h1 = tf.nn.relu(z1)\n",
        "      batch_mean1, batch_var1 = tf.nn.moments(z1_h1, [0])\n",
        "      z1_BN = (z1_h1 - batch_mean1) / tf.sqrt(batch_var1 + self.epsilon)\n",
        "      h1 = self.scale1 * z1_BN + self.bias1\n",
        "      \n",
        "      z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "      z2_h2 = tf.nn.relu(z2)\n",
        "      batch_mean2, batch_var2 = tf.nn.moments(z2_h2, [0])\n",
        "      z2_BN = (z2_h2 - batch_mean2) / tf.sqrt(batch_var2 + self.epsilon)\n",
        "      h2 = self.scale2 * z2_BN + self.bias2\n",
        "      \n",
        "      z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "      z3_h3 = tf.nn.relu(z3)\n",
        "      batch_mean3, batch_var3 = tf.nn.moments(z3_h3, [0])\n",
        "      z3_BN = (z3_h3 - batch_mean3) / tf.sqrt(batch_var3 + self.epsilon)\n",
        "      h3 = self.scale3 * z3_BN + self.bias3\n",
        "\n",
        "    else:\n",
        "      z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "      z1_h1 = tf.nn.relu(z1)\n",
        "      z1_BN = (z1_h1 - mean1) / tf.sqrt(var1 + self.epsilon)\n",
        "      h1 = self.scale1 * z1_BN + self.bias1\n",
        "      \n",
        "      z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "      z2_h2 = tf.nn.relu(z2)\n",
        "      z2_BN = (z2_h2 - mean2) / tf.sqrt(var2 + self.epsilon)\n",
        "      h2 = self.scale2 * z2_BN + self.bias2\n",
        "      \n",
        "      z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "      z3_h3 = tf.nn.relu(z3)\n",
        "      z3_BN = (z3_h3 - mean3) / tf.sqrt(var3 + self.epsilon)\n",
        "      h3 = self.scale3 * z3_BN + self.bias3\n",
        "\n",
        "    # Compute output\n",
        "    output = tf.matmul(h3, self.W4) + self.b4\n",
        "    \n",
        "    #Now consider two things , First look at inbuild loss functions if they work with softmax or not and then change this \n",
        "    # Second add tf.Softmax(output) and then return this variable\n",
        "    return (output)\n",
        "\n",
        "#  def stderr(self,y_pred):\n",
        "#     \"\"\"\n",
        "#      Calculate standard error\n",
        "#      \"\"\"\n",
        "#     y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "#     std_dev = np.std(y_pred_tf) #Calculates standard deviation\n",
        "#     std_err = std_dev/sqrt(len(y_pred_tf))\n",
        "#     return std_err \n",
        "\n",
        "\n",
        "#  def var(self,y_pred):\n",
        "#     \"\"\"\n",
        "#      Calculate variance \n",
        "#      \"\"\"\n",
        "#     y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "#     std_dev = np.std(y_pred_tf) #Calculates standard deviation\n",
        "#     variance = (std_dev**2) # calculate variance\n",
        "#     return variance "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Validation"
      ],
      "metadata": {
        "id": "Yz9HHx6BowjS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "cN5jOZnKujb9",
        "outputId": "9553502c-4569-4bd9-f9de-03122341684d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Train Accuracy: 0.8552\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.006404358520507813 \n",
            "\n",
            "Validation Accuracy: 0.8411\n",
            "\n",
            "Train Accuracy: 0.8807\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.005052757263183594 \n",
            "\n",
            "Validation Accuracy: 0.8629\n",
            "\n",
            "Train Accuracy: 0.8807\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.0048899420166015626 \n",
            "\n",
            "Validation Accuracy: 0.8526\n",
            "\n",
            "Train Accuracy: 0.8975\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.004215007934570312 \n",
            "\n",
            "Validation Accuracy: 0.8661\n",
            "\n",
            "Train Accuracy: 0.8938\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.004299261474609375 \n",
            "\n",
            "Validation Accuracy: 0.8586\n",
            "\n",
            "Train Accuracy: 0.9008\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.004078229370117188 \n",
            "\n",
            "Validation Accuracy: 0.8629\n",
            "\n",
            "Train Accuracy: 0.9243\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.0031858657836914064 \n",
            "\n",
            "Validation Accuracy: 0.8760\n",
            "\n",
            "Train Accuracy: 0.9064\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.003711279296875 \n",
            "\n",
            "Validation Accuracy: 0.8572\n",
            "\n",
            "Train Accuracy: 0.9223\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.003290724792480469 \n",
            "\n",
            "Validation Accuracy: 0.8736\n",
            "\n",
            "Train Accuracy: 0.9295\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.002987183837890625 \n",
            "\n",
            "Validation Accuracy: 0.8709\n",
            "\n",
            "Total time taken (in seconds): 360.08\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAY80lEQVR4nO3df2xV553n8feHn62zXSeh3ohA8LUmbitSVFp52HY7qjTxZELaEjISmrpyV2iE5BkJup3NalqQ/9hJJK/KqLPQP9JInvwoSr1DEDtVTFabNIVIo5FmADOhIZCgWBCD2SS4CfFuiwSYfPeP+5Dc417wwTaca9/PS0I+53ue89znXAl/fO5zzrmKCMzMzK6YU/QAzMystjgYzMwsw8FgZmYZDgYzM8twMJiZWca8ogcwHT796U9HqVQqehhmZjPKoUOHfh0RTePrsyIYSqUSAwMDRQ/DzGxGkTRUrZ7royRJqyUdlzQoaXOV7QslPZu275dUqti2JdWPS7q/on6rpN2S3pD0uqSvpPpfSzoj6XD69/XrPVgzM5u8Cc8YJM0FHgPuA4aBg5L6I+JYRbMNwLmIuFtSB7AV+Jak5UAHcA9wJ/BLSZ+JiMvAj4EXImKdpAVAQ0V/2yLiR9NxgGZmdn3ynDGsAgYj4kREXAR2AmvHtVkL7EjLu4F2SUr1nRFxISJOAoPAKkmNwNeAJwEi4mJEfDD1wzEzs6nKEwxLgNMV68OpVrVNRIwBo8Cia+zbAowAT0t6RdITkm6paLdJ0quSnpJ0W7VBSeqSNCBpYGRkJMdhmJlZHkVdrjoP+BLweER8EfgtcGXu4nHg94CVwNvA31brICJ6I6ItItqamn5nUt3MzCYpTzCcAe6qWF+aalXbSJoHNALvXWPfYWA4Ivan+m7KQUFEvBsRlyPiQ+DvKH+UNe36jvRR2l5iziNzKG0v0Xek70a8jJnZjJMnGA4CrZJa0iRxB9A/rk0/sD4trwP2Rfmxrf1AR7pqqQVoBQ5ExDvAaUmfTfu0A8cAJC2u6PdPgNcmcVzX1Hekj649XQyNDhEEQ6NDdO3pcjiYmZHjqqSIGJO0CXgRmAs8FRFHJT0KDEREP+VJ5GckDQLvUw4PUrtdlH/pjwEb0xVJAN8F+lLYnAD+LNX/RtJKIIC3gD+fnkP9WPfebs5fOp+pnb90nu693XSu6JzulzMzm1E0G76Poa2tLa7nBrc5j8wh+N3jFuLD//rhdA7NzKxmSToUEW3j63X5rKRljcuuq25mVk/qMhh62ntomN+QqTXMb6CnvaegEZmZ1Y66DIbOFZ30rumlubEZIZobm+ld0+v5BTMz6nSOwczMPMdgZmY5ORjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaWkSsYJK2WdFzSoKTNVbYvlPRs2r5fUqli25ZUPy7p/or6rZJ2S3pD0uuSvpLqt0t6SdKb6edtUz9MMzPLa8JgkDQXeAx4AFgOfFvS8nHNNgDnIuJuYBuwNe27HOgA7gFWAz9J/QH8GHghIj4HfAF4PdU3A3sjohXYm9bNzOwmyXPGsAoYjIgTEXER2AmsHddmLbAjLe8G2iUp1XdGxIWIOAkMAqskNQJfA54EiIiLEfFBlb52AA9N7tDMzGwy8gTDEuB0xfpwqlVtExFjwCiw6Br7tgAjwNOSXpH0hKRbUps7IuLttPwOcEf+wzEzs6kqavJ5HvAl4PGI+CLwW6p8ZBTlL6Su+qXUkrokDUgaGBkZuaGDNTOrJ3mC4QxwV8X60lSr2kbSPKAReO8a+w4DwxGxP9V3Uw4KgHclLU59LQbOVhtURPRGRFtEtDU1NeU4DDMzyyNPMBwEWiW1SFpAeTK5f1ybfmB9Wl4H7Et/7fcDHemqpRagFTgQEe8ApyV9Nu3TDhyr0td64LlJHJeZmU3SvIkaRMSYpE3Ai8Bc4KmIOCrpUWAgIvopTyI/I2kQeJ9yeJDa7aL8S38M2BgRl1PX3wX6UticAP4s1X8I7JK0ARgC/nSajtXMzHJQ+Q/7ma2trS0GBgaKHoaZ2Ywi6VBEtI2v+85nMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaWkSsYJK2WdFzSoKTNVbYvlPRs2r5fUqli25ZUPy7p/or6W5KOSDosaaCi/teSzqT6YUlfn9ohmpnZ9Zg3UQNJc4HHgPuAYeCgpP6IOFbRbANwLiLultQBbAW+JWk50AHcA9wJ/FLSZyLictrvDyPi11VedltE/Gjyh2VmZpOV54xhFTAYESci4iKwE1g7rs1aYEda3g20S1Kq74yICxFxEhhM/ZmZWY3KEwxLgNMV68OpVrVNRIwBo8CiCfYN4BeSDknqGtffJkmvSnpK0m25jsTMzKZFkZPPfxARXwIeADZK+lqqPw78HrASeBv422o7S+qSNCBpYGRk5KYM2MysHuQJhjPAXRXrS1OtahtJ84BG4L1r7RsRV36eBX5O+ogpIt6NiMsR8SHwd1zlo6eI6I2Itohoa2pqynEYZmaWR55gOAi0SmqRtIDyZHL/uDb9wPq0vA7YFxGR6h3pqqUWoBU4IOkWSZ8CkHQL8MfAa2l9cUW/f3KlbmZmN8eEVyVFxJikTcCLwFzgqYg4KulRYCAi+oEngWckDQLvUw4PUrtdwDFgDNgYEZcl3QH8vDw/zTzgf0TEC+kl/0bSSspzEG8Bfz59h2tmZhNR+Q/7ma2trS0GBgYmbmhmZh+RdCgi2sbXfeezmZllOBjMzCzDwWBmZhkOBjMzy3AwmJlZhoPBzMwyHAxmZpbhYDAzswwHg5mZZTgYzMwsw8FgZmYZDgYzM8twMJiZWYaDwczMMhwMZmaW4WAwM7MMB4OZmWU4GArWd6SP0vYScx6ZQ2l7ib4jfUUPyczq3ITf+Ww3Tt+RPrr2dHH+0nkAhkaH6NrTBUDnis4ih2ZmdcxnDAXq3tv9UShccf7Sebr3dhc0IjOznMEgabWk45IGJW2usn2hpGfT9v2SShXbtqT6cUn3V9TfknRE0mFJAxX12yW9JOnN9PO2qR1i7To1euq66mZmN8OEwSBpLvAY8ACwHPi2pOXjmm0AzkXE3cA2YGvadznQAdwDrAZ+kvq74g8jYmVEtFXUNgN7I6IV2JvWZ6Vljcuuq25mdjPkOWNYBQxGxImIuAjsBNaOa7MW2JGWdwPtkpTqOyPiQkScBAZTf9dS2dcO4KEcY5yRetp7aJjfkKk1zG+gp72noBGZmeULhiXA6Yr14VSr2iYixoBRYNEE+wbwC0mHJHVVtLkjIt5Oy+8Ad1QblKQuSQOSBkZGRnIcRu3pXNFJ75pemhubEaK5sZneNb2eeDazQhV5VdIfRMQZSf8OeEnSGxHxj5UNIiIkRbWdI6IX6AVoa2ur2mYm6FzR6SAws5qS54zhDHBXxfrSVKvaRtI8oBF471r7RsSVn2eBn/PxR0zvSlqc+loMnM1/OGZmNlV5guEg0CqpRdICypPJ/ePa9APr0/I6YF9ERKp3pKuWWoBW4ICkWyR9CkDSLcAfA69V6Ws98NzkDs3MzCZjwmBIcwabgBeB14FdEXFU0qOSHkzNngQWSRoEHiZdSRQRR4FdwDHgBWBjRFymPG/wT5J+BRwA/ldEvJD6+iFwn6Q3gT9K63aD+Q5sM7tC5T/sZ7a2trYYGBiYuKFVNf4ObChfHeWJcLPZTdKhcbcLAL7z2fAd2GaW5WAw34FtZhkOBvMd2GaW4WAw34FtZhkOBvMd2GaW4auSzMzqlK9KMjOzXBwMZmaW4WAwM7MMB4OZmWU4GMzMLMPBYGZmGQ4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhm5gkHSaknHJQ1K2lxl+0JJz6bt+yWVKrZtSfXjku4ft99cSa9Ier6i9lNJJyUdTv9WTv7wzMzses2bqIGkucBjwH3AMHBQUn9EHKtotgE4FxF3S+oAtgLfkrQc6ADuAe4EfinpMxFxOe33PeB14N+Oe9m/iojdUzkwMzObnDxnDKuAwYg4EREXgZ3A2nFt1gI70vJuoF2SUn1nRFyIiJPAYOoPSUuBbwBPTP0wzMxsuuQJhiXA6Yr14VSr2iYixoBRYNEE+24Hvg98WOU1eyS9KmmbpIXVBiWpS9KApIGRkZEch2FmZnkUMvks6ZvA2Yg4VGXzFuBzwO8DtwM/qNZHRPRGRFtEtDU1Nd24wZqZ1Zk8wXAGuKtifWmqVW0jaR7QCLx3jX2/Cjwo6S3KH03dK+lnABHxdpRdAJ4mffRkZmY3R55gOAi0SmqRtIDyZHL/uDb9wPq0vA7YFxGR6h3pqqUWoBU4EBFbImJpRJRSf/si4jsAkhannwIeAl6b0hGamdl1mfCqpIgYk7QJeBGYCzwVEUclPQoMREQ/8CTwjKRB4H3Kv+xJ7XYBx4AxYGPFFUlX0yepCRBwGPiLSR6bmZlNgsp/2M9sbW1tMTAwUPQwzMxmFEmHIqJtfN13PpuZWYaDwczMMhwMZmaW4WAwG6fvSB+l7SXmPDKH0vYSfUf6ih6S2U014VVJZvWk70gfXXu6OH/pPABDo0N07ekCoHNFZ5FDM7tpfMZgVqF7b/dHoXDF+Uvn6d7bXdCIzG4+B4NZhVOjp66rbjYbORjMKixrXHZddbPZyMFgNaXoid+e9h4a5jdkag3zG+hp77mp4zArkoPBasaVid+h0SGC+Gji92aGQ+eKTnrX9NLc2IwQzY3N9K7p9cSz1RU/EsNqRml7iaHRod+pNzc289ZfvnXzB2Q2y/mRGFbzPPFrVhscDFYzPPFrVhscDFYzPPFrVhscDFYzPPFrVhs8+WxmVqc8+Ww2wxR9T4fVLz9Ez6wG+WF+ViSfMZjVID/Mz4rkYDCrQb6nw4qUKxgkrZZ0XNKgpM1Vti+U9Gzavl9SqWLbllQ/Lun+cfvNlfSKpOcrai2pj8HU54LJH57ZzOR7OqxIEwaDpLnAY8ADwHLg25KWj2u2ATgXEXcD24Ctad/lQAdwD7Aa+Enq74rvAa+P62srsC31dS71bVZXfE+HFSnPGcMqYDAiTkTERWAnsHZcm7XAjrS8G2iXpFTfGREXIuIkMJj6Q9JS4BvAE1c6Sfvcm/og9fnQZA7MbCbzPR1WpDxXJS0BTlesDwP//mptImJM0iiwKNX/Zdy+S9LyduD7wKcqti8CPoiIsSrtMyR1AV0Ay5b59Npmn84VnQ4CK0Qhk8+SvgmcjYhDk+0jInojoi0i2pqamqZxdGZm9S1PMJwB7qpYX5pqVdtImgc0Au9dY9+vAg9KeovyR1P3SvpZ2ufW1MfVXsvMzG6gPMFwEGhNVwstoDyZ3D+uTT+wPi2vA/ZF+Vkb/UBHumqpBWgFDkTElohYGhGl1N++iPhO2ufl1Aepz+emcHxmZnadJgyG9Hn/JuBFylcQ7YqIo5IelfRgavYksEjSIPAwsDntexTYBRwDXgA2RsTlCV7yB8DDqa9FqW8zM7tJ/BA9M7M65YfomZlZLg4GMzPLcDCYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmM0LfkT5K20vMeWQOpe0l+o70FT2kWSvPY7fNzArVd6SPrj1dH30P9tDoEF17ugD8aPIbwGcMZlbzuvd2fxQKV5y/dJ7uvd0FjWh2czCYWc07NXrquuo2NQ4GM6t5yxqrf0vj1eo2NQ4GM6t5Pe09NMxvyNQa5jfQ095T0IhmNweDmdW8zhWd9K7ppbmxGSGaG5vpXdPriecbxN/HYGZWp/x9DGZmlouDwczMMhwMZmaW4WAwM7OMXMEgabWk45IGJW2usn2hpGfT9v2SShXbtqT6cUn3p9onJB2Q9CtJRyU9UtH+p5JOSjqc/q2c+mGamVleEz4rSdJc4DHgPmAYOCipPyKOVTTbAJyLiLsldQBbgW9JWg50APcAdwK/lPQZ4AJwb0T8RtJ84J8k/e+I+JfU319FxO7pOkgzM8svzxnDKmAwIk5ExEVgJ7B2XJu1wI60vBtol6RU3xkRFyLiJDAIrIqy36T289O/mX/drJnZLJAnGJYApyvWh1OtapuIGANGgUXX2lfSXEmHgbPASxGxv6Jdj6RXJW2TtLDaoCR1SRqQNDAyMpLjMMzMLI/CJp8j4nJErASWAqskfT5t2gJ8Dvh94HbgB1fZvzci2iKiramp6aaM2cysHuQJhjPAXRXrS1OtahtJ84BG4L08+0bEB8DLwOq0/nb6qOkC8DTlj7LMzOwmyRMMB4FWSS2SFlCeTO4f16YfWJ+W1wH7ovysjX6gI1211AK0AgckNUm6FUDSJylPbL+R1hennwIeAl6bygGamdn1mfCqpIgYk7QJeBGYCzwVEUclPQoMREQ/8CTwjKRB4H3K4UFqtws4BowBGyPicvrlvyNd8TQH2BURz6eX7JPUBAg4DPzFdB6wmZldmx+iZ2ZWp/wQPTMzy8XBYGZmGQ4GMzPLcDCYmV2HviN9lLaXmPPIHErbS/Qd6St6SNNuwquSzMysrO9IH117ujh/6TwAQ6NDdO3pAphVXzPqMwYzs5y693Z/FApXnL90nu693QWN6MZwMJiZ5XRq9NR11WcqB4OZWU7LGpddV32mcjCYmeXU095Dw/yGTK1hfgM97T0FjejGcDCY2TXVw1U4eXWu6KR3TS/Njc0I0dzYTO+a3lk18Qx+JIaZXcP4q3Cg/BfybPxlWI/8SAwzu271chWOZTkYzOyq6uUqHMtyMJjZVdXLVTiW5WAws6uql6twLMvBYGZXVS9X4ViWr0oyM6tTvirJzMxycTCYmVmGg8HMzDJyBYOk1ZKOSxqUtLnK9oWSnk3b90sqVWzbkurHJd2fap+QdEDSryQdlfRIRfuW1Mdg6nPB1A/TzMzymjAYJM0FHgMeAJYD35a0fFyzDcC5iLgb2AZsTfsuBzqAe4DVwE9SfxeAeyPiC8BKYLWkL6e+tgLbUl/nUt9mZlbhRj7DKs8ZwypgMCJORMRFYCewdlybtcCOtLwbaJekVN8ZERci4iQwCKyKst+k9vPTv0j73Jv6IPX50CSPzcxsVrryDKuh0SGC+Oib5KYrHPIEwxLgdMX6cKpVbRMRY8AosOha+0qaK+kwcBZ4KSL2p30+SH1c7bVI+3dJGpA0MDIykuMwzMxmhxv9DKvCJp8j4nJErASWAqskff469++NiLaIaGtqaroxgzQzq0E3+hlWeYLhDHBXxfrSVKvaRtI8oBF4L8++EfEB8DLlOYj3gFtTH1d7LTOzunajn2GVJxgOAq3paqEFlCeT+8e16QfWp+V1wL4o31LdD3Skq5ZagFbggKQmSbcCSPokcB/wRtrn5dQHqc/nJn94Zmazz41+htWEwZA+798EvAi8DuyKiKOSHpX0YGr2JLBI0iDwMLA57XsU2AUcA14ANkbEZWAx8LKkVykHz0sR8Xzq6wfAw6mvRalvMzNLbvQzrPysJDOzOuVnJZmZWS4OBjMzy3AwmJlZhoPBzMwyHAxmZpYxK65KkjQCDBU9jin6NPDrogdRQ/x+fMzvRZbfj6ypvB/NEfE7j46YFcEwG0gaqHbZWL3y+/ExvxdZfj+ybsT74Y+SzMwsw8FgZmYZDoba0Vv0AGqM34+P+b3I8vuRNe3vh+cYzMwsw2cMZmaW4WAwM7MMB0PBJN0l6WVJxyQdlfS9osdUtPS1r69Ien7i1rObpFsl7Zb0hqTXJX2l6DEVRdJ/Tv9HXpP095I+UfSYbiZJT0k6K+m1itrtkl6S9Gb6edt0vJaDoXhjwH+JiOXAl4GNkpYXPKaifY/yd38Y/Bh4ISI+B3yBOn1fJC0B/hPQFhGfB+ZS/tKwevJTyt90WWkzsDciWoG9aX3KHAwFi4i3I+Jf0/L/o/wff0mxoyqOpKXAN4Anih5L0SQ1Al8jfVlVRFxMX4Vbr+YBn0xf/dsA/J+Cx3NTRcQ/Au+PK68FdqTlHcBD0/FaDoYaIqkEfBHYX+xICrUd+D7wYdEDqQEtwAjwdPpo7QlJtxQ9qCJExBngR8Ap4G1gNCJ+UeyoasIdEfF2Wn4HuGM6OnUw1AhJ/wb4n8BfRsT/LXo8RZD0TeBsRBwqeiw1Yh7wJeDxiPgi8Fum6aOCmSZ9dr6WcljeCdwi6TvFjqq2RPneg2m5/8DBUAMkzaccCn0R8Q9Fj6dAXwUelPQWsBO4V9LPih1SoYaB4Yi4cga5m3JQ1KM/Ak5GxEhEXAL+AfgPBY+pFrwraTFA+nl2Ojp1MBRMkih/hvx6RPz3osdTpIjYEhFLI6JEeWJxX0TU7V+FEfEOcFrSZ1OpHThW4JCKdAr4sqSG9H+mnTqdiB+nH1ifltcDz01Hpw6G4n0V+I+U/zo+nP59vehBWc34LtAn6VVgJfDfCh5PIdJZ027gX4EjlH931dWjMST9PfDPwGclDUvaAPwQuE/Sm5TPqn44La/lR2KYmVklnzGYmVmGg8HMzDIcDGZmluFgMDOzDAeDmZllOBjMzCzDwWBmZhn/H2SFrIZPiEKPAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "###################### before\n",
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP_NB(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type = 'before', epsilon = 1e-3, device = 'cpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.SGD(learning_rate = 0.1)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "  \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(batch_size)\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  \n",
        "  # training \n",
        "  for inputs, outputs in train_ds:\n",
        "    #preds = mlp_on_cpu.forward(inputs, training = True, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti, training = True)\n",
        "\n",
        "  # layer means and vars given all data\n",
        "  mean1, var1, mean2, var2, mean3, var3 = mlp_on_cpu.layer_mean_var(X_train)\n",
        "\n",
        "  # training loss\n",
        "  for inputs, outputs in train_ds:\n",
        "    preds = mlp_on_cpu.forward(inputs, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epoch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "c2gTBX8zVMcC",
        "outputId": "676724b7-3359-4698-b6fb-61e7fdadfbb8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Train Accuracy: 0.8657\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.00580166748046875 \n",
            "\n",
            "Validation Accuracy: 0.8494\n",
            "\n",
            "Train Accuracy: 0.8766\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.005262592163085937 \n",
            "\n",
            "Validation Accuracy: 0.8593\n",
            "\n",
            "Train Accuracy: 0.8782\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.0051205224609375 \n",
            "\n",
            "Validation Accuracy: 0.8512\n",
            "\n",
            "Train Accuracy: 0.8945\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.004328527221679687 \n",
            "\n",
            "Validation Accuracy: 0.8711\n",
            "\n",
            "Train Accuracy: 0.9019\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.004092961120605469 \n",
            "\n",
            "Validation Accuracy: 0.8709\n",
            "\n",
            "Train Accuracy: 0.8969\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.004291982421875 \n",
            "\n",
            "Validation Accuracy: 0.8671\n",
            "\n",
            "Train Accuracy: 0.9136\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.0036248968505859375 \n",
            "\n",
            "Validation Accuracy: 0.8767\n",
            "\n",
            "Train Accuracy: 0.9032\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.003959103393554687 \n",
            "\n",
            "Validation Accuracy: 0.8633\n",
            "\n",
            "Train Accuracy: 0.9159\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.003530745849609375 \n",
            "\n",
            "Validation Accuracy: 0.8775\n",
            "\n",
            "Train Accuracy: 0.9030\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.004044321594238281 \n",
            "\n",
            "Validation Accuracy: 0.8584\n",
            "\n",
            "Total time taken (in seconds): 397.49\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATaElEQVR4nO3df4xd5X3n8ffHPyA1m04SYiGCscdanEam1tJoynY3VaTGm0J+gFMJNa4mFX8gzVaCbbrdbYLlfxYkr8Kqu3j/oJVmAwlKpjXI2yhDpIYkJlK10hYYN2zMj1oZATamJDiEeLdrCTD57h/3GO6ZHTPX9njO2Pf9kkZzz3Oe89znXMn+zDnPuc+TqkKSpJNWdN0BSdLyYjBIkloMBklSi8EgSWoxGCRJLau67sBieP/731+jo6Ndd0OSziv79+//aVWtnVt+QQTD6OgoMzMzXXdDks4rSQ7NV+6tJElSi8EgSWoxGCRJLQaDJKnFYJAktQxtMEwdmGJ09ygr7ljB6O5Rpg5Mdd0lSVoWLojHVU/X1IEpJh6a4PgbxwE4dOwQEw9NADC+ZbzLrklS54byimHnvp1vhcJJx984zs59OzvqkSQtH0MZDIePHT6tckkaJkMZDOtH1p9WuSQNk6EMhl1bd7Fm9ZpW2ZrVa9i1dVdHPZKk5WMog2F8yziTN0yyYWQDIWwY2cDkDZMOPEsSkAthzeexsbFyEj1JOj1J9lfV2NzyobxikCSdmsEgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJahkoGJJcn+Rgktkkt8+z/+IkDzT7H00y2rdvR1N+MMl1feXPJzmQ5IkkM33l/yHJi035E0k+eXanKEk6HasWqpBkJXAP8HHgCPB4kumqerqv2i3Aq1V1VZLtwF3AZ5NsBrYDVwMfAL6X5INV9WZz3G9V1U/nedu7q+pPz/y0JElnapArhmuB2ap6tqpeB/YA2+bU2Qbc37zeC2xNkqZ8T1W9VlXPAbNNe5KkZWqQYLgCeKFv+0hTNm+dqjoBHAMuXeDYAr6TZH+SiTnt3Zbkh0nuS/Legc5EkrQouhx8/s2q+jDwCeDWJB9tyv8c+KfANcBLwH+e7+AkE0lmkswcPXp0STosScNgkGB4Ebiyb3tdUzZvnSSrgBHglXc6tqpO/n4Z+AbNLaaq+klVvVlVvwD+G6e49VRVk1U1VlVja9euHeA0JEmDGCQYHgc2JdmY5CJ6g8nTc+pMAzc3r28CHqmqasq3N08tbQQ2AY8luSTJuwGSXAL8NvBks315X7u/c7JckrQ0FnwqqapOJLkNeBhYCdxXVU8luROYqapp4F7ga0lmgZ/RCw+aeg8CTwMngFur6s0klwHf6I1Pswr4i6r6dvOW/ynJNfTGIJ4H/vXina4kaSHp/WF/fhsbG6uZmZmFK0qS3pJkf1WNzS33m8+SpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovB0LGpA1OM7h5lxR0rGN09ytSBqa67JGnILTjtts6dqQNTTDw0wfE3jgNw6NghJh7qrXI6vmW8y65JGmJeMXRo576db4XCScffOM7OfTs76pEkGQydOnzs8GmVS9JSMBg6tH5k/WmVS9JSMBg6tGvrLtasXtMqW7N6Dbu27uqoR5JkMHRqfMs4kzdMsmFkAyFsGNnA5A2TDjxL6pRrPkvSkHLNZ0nSQAwGSVKLwSBJajEYJEktBoMkqcVgEOBkfpLe5iR6cjI/SS1eMcjJ/CS1GAxyMj9JLQaDnMxPUovBICfzk9RiMMjJ/CS1OImeJA0pJ9GTJA3EYJAktQwUDEmuT3IwyWyS2+fZf3GSB5r9jyYZ7du3oyk/mOS6vvLnkxxI8kSSmb7y9yX5bpIfNb/fe3anKEk6HQsGQ5KVwD3AJ4DNwO8l2Tyn2i3Aq1V1FXA3cFdz7GZgO3A1cD3wZ017J/1WVV0z5x7X7cC+qtoE7Gu2JUlLZJArhmuB2ap6tqpeB/YA2+bU2Qbc37zeC2xNkqZ8T1W9VlXPAbNNe++kv637gc8M0EdJ0iIZJBiuAF7o2z7SlM1bp6pOAMeASxc4toDvJNmfZKKvzmVV9VLz+sfAZfN1KslEkpkkM0ePHh3gNCRJg+hy8Pk3q+rD9G5R3Zrko3MrVO9Z2nmfp62qyaoaq6qxtWvXnuOuStLwGCQYXgSu7Nte15TNWyfJKmAEeOWdjq2qk79fBr7B27eYfpLk8qaty4GXBz8dSdLZGiQYHgc2JdmY5CJ6g8nTc+pMAzc3r28CHmn+2p8GtjdPLW0ENgGPJbkkybsBklwC/Dbw5Dxt3Qx888xOTZJ0JhZcj6GqTiS5DXgYWAncV1VPJbkTmKmqaeBe4GtJZoGf0QsPmnoPAk8DJ4Bbq+rNJJcB3+iNT7MK+Iuq+nbzll8CHkxyC3AI+N1FPF9J0gKcEkOShpRTYkiSBmIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgYKhiTXJzmYZDbJ7fPsvzjJA83+R5OM9u3b0ZQfTHLdnONWJvlBkm/1lX01yXNJnmh+rjnz05Mkna5VC1VIshK4B/g4cAR4PMl0VT3dV+0W4NWquirJduAu4LNJNgPbgauBDwDfS/LBqnqzOe7zwDPAL8952z+pqr1nc2KSpDMzyBXDtcBsVT1bVa8De4Btc+psA+5vXu8FtiZJU76nql6rqueA2aY9kqwDPgV8+exPQ5K0WAYJhiuAF/q2jzRl89apqhPAMeDSBY7dDXwB+MU877kryQ+T3J3k4gH6KElaJJ0MPif5NPByVe2fZ/cO4EPArwPvA754ijYmkswkmTl69Oi566wkDZlBguFF4Mq+7XVN2bx1kqwCRoBX3uHYjwA3Jnme3q2pjyX5OkBVvVQ9rwFfobn1NFdVTVbVWFWNrV27doDTkCQNYpBgeBzYlGRjkovoDSZPz6kzDdzcvL4JeKSqqinf3jy1tBHYBDxWVTuqal1VjTbtPVJVnwNIcnnzO8BngCfP6gwlSadlwaeSqupEktuAh4GVwH1V9VSSO4GZqpoG7gW+lmQW+Bm9/+xp6j0IPA2cAG7teyLpVKaSrAUCPAH8wRmemyTpDKT3h/35bWxsrGZmZrruhiSdV5Lsr6qxueV+81mS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWDQsjJ1YIrR3aOsuGMFo7tHmTow1XWXpKGz4BfcpKUydWCKiYcmOP7GcQAOHTvExEMTAIxvGe+ya9JQ8YpBy8bOfTvfCoWTjr9xnJ37dnbUI2k4GQxaNg4fO3xa5ZLODYNBy8b6kfWnVX6hc7xFXTEYtGzs2rqLNavXtMrWrF7Drq27OupRd06Otxw6doii3hpvMRy0FAwGLRvjW8aZvGGSDSMbCGHDyAYmb5gcyoFnx1vUJZ9K0rIyvmV8KINgLsdb1CWvGKRlyPEWdclgkJYhx1vUJYNBWoYcb1GXXMFNkoaUK7hJkgZiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIc7hAjoad025LfU4ukHNyLYSTC+QAzlOkoeEVg9THBXIkg0FqcYEcyWCQWlwgR+eLczkWZjBIfVwgR+eDk2Nhh44doqi3xsIWKxwMBqmPC+TofHCux8J8KkmaY3zLuEGgZe1cj4UNdMWQ5PokB5PMJrl9nv0XJ3mg2f9oktG+fTua8oNJrptz3MokP0jyrb6yjU0bs02bF5356UnShedcj4UtGAxJVgL3AJ8ANgO/l2TznGq3AK9W1VXA3cBdzbGbge3A1cD1wJ817Z30eeCZOW3dBdzdtPVq07YkqXGux8IGuWK4Fpitqmer6nVgD7BtTp1twP3N673A1iRpyvdU1WtV9Rww27RHknXAp4Avn2ykOeZjTRs0bX7mTE5Mki5U53osbJAxhiuAF/q2jwD//FR1qupEkmPApU3538459orm9W7gC8C7+/ZfCvy8qk7MU78lyQQwAbB+vY8SShou53IsrJOnkpJ8Gni5qvafaRtVNVlVY1U1tnbt2kXsnaTlyDmsls4gVwwvAlf2ba9ryuarcyTJKmAEeOUdjr0RuDHJJ4F3Ab+c5OvA7wPvSbKquWqY770kDRnnsFpag1wxPA5sap4WuojeYPL0nDrTwM3N65uAR6qqmvLtzVNLG4FNwGNVtaOq1lXVaNPeI1X1ueaY7zdt0LT5zbM4P0kXAOewWloLBkPzl/ttwMP0niB6sKqeSnJnkhubavcClyaZBf4YuL059ingQeBp4NvArVX15gJv+UXgj5u2Lm3aljTEnMNqaaX3R/r5bWxsrGZmZrruhqRzZHT3KIeOHfr/yjeMbOD5P3p+6Tt0gUiyv6rG5pY7JYakZc85rJaWwSBp2XMOq6XlrSRJGlLeSpIkDcRgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSe9o6sAUo7tHWXHHCkZ3jzJ1YKrrLukcW9V1ByQtX1MHpph4aILjbxwH4NCxQ0w8NAHA+JbxLrumc8grBkmntHPfzrdC4aTjbxxn576dHfVIS8FgkHRKh48dPq1yXRgMBkmntH5k/WmV68JgMEg6pV1bd7Fm9ZpW2ZrVa9i1dVdHPdJSMBgkndL4lnEmb5hkw8gGQtgwsoHJGyYdeL7Apaq67sNZGxsbq5mZma67IUnnlST7q2psbrlXDJJ0Gobhex1+j0GSBjQs3+sY6IohyfVJDiaZTXL7PPsvTvJAs//RJKN9+3Y05QeTXNeUvSvJY0n+V5KnktzRV/+rSZ5L8kTzc83Zn6Yknb1h+V7HglcMSVYC9wAfB44AjyeZrqqn+6rdArxaVVcl2Q7cBXw2yWZgO3A18AHge0k+CLwGfKyq/jHJauB/JPnrqvrbpr0/qaq9i3WSkrQYhuV7HYNcMVwLzFbVs1X1OrAH2Danzjbg/ub1XmBrkjTle6rqtap6DpgFrq2ef2zqr25+zv9RcEkXtGH5XscgwXAF8ELf9pGmbN46VXUCOAZc+k7HJlmZ5AngZeC7VfVoX71dSX6Y5O4kF5/G+UjSOTMs3+vo7Kmkqnqzqq4B1gHXJvnVZtcO4EPArwPvA7443/FJJpLMJJk5evTokvRZ0nAblu91DPJU0ovAlX3b65qy+eocSbIKGAFeGeTYqvp5ku8D1wNPVtVLza7XknwF+PfzdaqqJoFJ6H2PYYDzkKSzNr5l/IILgrkGuWJ4HNiUZGOSi+gNJk/PqTMN3Ny8vgl4pHrfnJsGtjdPLW0ENgGPJVmb5D0ASX6J3sD23zfblze/A3wGePJsTlCSdHoWvGKoqhNJbgMeBlYC91XVU0nuBGaqahq4F/haklngZ/TCg6beg8DTwAng1qp6s/nP//7miacVwINV9a3mLaeSrAUCPAH8wWKesCTpnTklhiQNKafEkCQNxGCQJLVcELeSkhwFDnXdj7P0fuCnXXdiGfHzeJufRZufR9vZfB4bqmrt3MILIhguBElm5rvXN6z8PN7mZ9Hm59F2Lj4PbyVJkloMBklSi8GwfEx23YFlxs/jbX4WbX4ebYv+eTjGIElq8YpBktRiMEiSWgyGjiW5Msn3kzzdLHP6+a771LVmrY4fJPnWwrUvbEnek2Rvkr9P8kySf9F1n7qS5N82/0aeTPKXSd7VdZ+WUpL7kryc5Mm+svcl+W6SHzW/37sY72UwdO8E8O+qajPwG8CtzZKow+zzwDNdd2KZ+K/At6vqQ8A/Y0g/lyRXAH8IjFXVr9Kb0HN7t71acl+ltzxBv9uBfVW1CdjXbJ81g6FjVfVSVf1d8/r/0PuHP3eFvKGRZB3wKeDLXfela0lGgI/Sm72Yqnq9qn7eba86tQr4pWbNlzXAP3TcnyVVVX9Db/bqfv3LKt9Pb6mCs2YwLCNJRoFfAx5955oXtN3AF4BfdN2RZWAjcBT4SnNr7ctJLum6U12oqheBPwUOAy8Bx6rqO932alm4rG9xsx8Dly1GowbDMpHknwD/HfijqvrfXfenC0k+DbxcVfu77ssysQr4MPDnVfVrwP9lkW4VnG+ae+fb6IXlB4BLknyu214tL83iaIvy/QODYRlIsppeKExV1V913Z8OfQS4McnzwB7gY0m+3m2XOnUEOFJVJ68g99ILimH0r4DnqupoVb0B/BXwLzvu03Lwk75VLy8HXl6MRg2GjjVLmN4LPFNV/6Xr/nSpqnZU1bqqGqU3sPhIVQ3tX4VV9WPghSS/0hRtpbca4jA6DPxGkjXNv5mtDOlA/Bz9yyrfDHxzMRo1GLr3EeD36f11/ETz88muO6Vl49/QW+72h8A1wH/suD+daK6a9gJ/Bxyg93/XUE2NkeQvgf8J/EqSI0luAb4EfDzJj+hdVX1pUd7LKTEkSf28YpAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS3/D25Tcv75brYQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "########################## after \n",
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP_NB(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type = 'after', epsilon = 1e-3, device = 'cpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.SGD(learning_rate = 0.1)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "  \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(batch_size)\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  \n",
        "  # training \n",
        "  for inputs, outputs in train_ds:\n",
        "    #preds = mlp_on_cpu.forward(inputs, training = True, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti, training = True)\n",
        "\n",
        "  # layer means and vars given all data\n",
        "  mean1, var1, mean2, var2, mean3, var3 = mlp_on_cpu.layer_mean_var(X_train)\n",
        "\n",
        "  # training loss\n",
        "  for inputs, outputs in train_ds:\n",
        "    preds = mlp_on_cpu.forward(inputs, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performance"
      ],
      "metadata": {
        "id": "gYNAumjZhKce"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_trials = 3\n",
        "\n",
        "test_accuracy = np.zeros([num_trials, 2]) # 0 for before 1 for after\n",
        "used_time = np.zeros([num_trials, 2])\n",
        "\n",
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10"
      ],
      "metadata": {
        "id": "kDAcfXlnhMQe"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for t in range(num_trials):\n",
        "  # Initialize model using CPU\n",
        "  mlp_on_cpu = MLP_NB(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type = 'before', epsilon = 1e-3, device = 'cpu')\n",
        "\n",
        "  time_start = time.time()\n",
        "  opti = tf.keras.optimizers.SGD(learning_rate = 0.1)\n",
        "\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "    print(t, epoch)\n",
        "    train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1178)).batch(batch_size)\n",
        "    \n",
        "    # training \n",
        "    for inputs, outputs in train_ds:\n",
        "      #preds = mlp_on_cpu.forward(inputs, training = True, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None)\n",
        "      mlp_on_cpu.backward(inputs, outputs, opti, training = True)\n",
        "\n",
        "  time_taken = time.time() - time_start\n",
        "      \n",
        "  used_time[t, 0] = time_taken\n",
        "      \n",
        "  # layer means and vars given all data\n",
        "  mean1, var1, mean2, var2, mean3, var3 = mlp_on_cpu.layer_mean_var(X_train)\n",
        "\n",
        "  # test sample\n",
        "  preds = mlp_on_cpu.forward(X_test, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_test, 1))\n",
        "  test_accuracy[t, 0] = tf.reduce_mean(tf.cast(correct_preds, 'float'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ug1ktdIchL5-",
        "outputId": "4a04d415-6a18-404a-9ceb-99a4cc857003"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 0\n",
            "0 1\n",
            "0 2\n",
            "0 3\n",
            "0 4\n",
            "0 5\n",
            "0 6\n",
            "0 7\n",
            "0 8\n",
            "0 9\n",
            "1 0\n",
            "1 1\n",
            "1 2\n",
            "1 3\n",
            "1 4\n",
            "1 5\n",
            "1 6\n",
            "1 7\n",
            "1 8\n",
            "1 9\n",
            "2 0\n",
            "2 1\n",
            "2 2\n",
            "2 3\n",
            "2 4\n",
            "2 5\n",
            "2 6\n",
            "2 7\n",
            "2 8\n",
            "2 9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for t in range(num_trials):\n",
        "  # Initialize model using CPU\n",
        "  mlp_on_cpu = MLP_NB(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, BN_type = 'after', epsilon = 1e-3, device = 'cpu')\n",
        "\n",
        "  time_start = time.time()\n",
        "  opti = tf.keras.optimizers.SGD(learning_rate = 0.1)\n",
        "\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "    print(t, epoch)\n",
        "    train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(178)).batch(batch_size)\n",
        "    \n",
        "    # training \n",
        "    for inputs, outputs in train_ds:\n",
        "      #preds = mlp_on_cpu.forward(inputs, training = True, mean1 = None, var1 = None, mean2 = None, var2 = None, mean3 = None, var3 = None)\n",
        "      mlp_on_cpu.backward(inputs, outputs, opti, training = True)\n",
        "\n",
        "  time_taken = time.time() - time_start\n",
        "      \n",
        "  used_time[t, 1] = time_taken\n",
        "      \n",
        "  # layer means and vars given all data\n",
        "  mean1, var1, mean2, var2, mean3, var3 = mlp_on_cpu.layer_mean_var(X_train)\n",
        "\n",
        "  # test sample\n",
        "  preds = mlp_on_cpu.forward(X_test, training = False, mean1 = mean1, var1 = var1, mean2 = mean2, var2 = var2, mean3 = mean3, var3 = var3)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_test, 1))\n",
        "  test_accuracy[t, 1] = tf.reduce_mean(tf.cast(correct_preds, 'float'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RxOZ8JpAvZT3",
        "outputId": "ebe51b21-6f60-4417-f5c8-8649f21f6c54"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 0\n",
            "0 1\n",
            "0 2\n",
            "0 3\n",
            "0 4\n",
            "0 5\n",
            "0 6\n",
            "0 7\n",
            "0 8\n",
            "0 9\n",
            "1 0\n",
            "1 1\n",
            "1 2\n",
            "1 3\n",
            "1 4\n",
            "1 5\n",
            "1 6\n",
            "1 7\n",
            "1 8\n",
            "1 9\n",
            "2 0\n",
            "2 1\n",
            "2 2\n",
            "2 3\n",
            "2 4\n",
            "2 5\n",
            "2 6\n",
            "2 7\n",
            "2 8\n",
            "2 9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "used_time"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2cEOeLfMs95x",
        "outputId": "18540b81-c0a0-4551-a116-bbe4f05d2a46"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[174.80074906, 191.88938785],\n",
              "       [191.17664671, 177.94594336],\n",
              "       [161.02315879, 187.18760562]])"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_accuracy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HZshsfSCvSPI",
        "outputId": "f4dadd73-af12-418e-b6da-4d3b94de1eb3"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.85549998, 0.86369997],\n",
              "       [0.86690003, 0.87589997],\n",
              "       [0.85439998, 0.87180001]])"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "tbLHWSTwvW28"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "IST597_HW4_MLP.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOfX2o/KeOWPg80ZV8JnAay",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}