Prototypical_Networks_in_TensorFlow_2_0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Prototypical Networks in TensorFlow 2.0.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayakpaul/TF-2.0-Hacks/blob/master/Prototypical_Networks_in_TensorFlow_2_0.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3weO3xt6-i1E",
        "colab_type": "text"
      },
      "source": [
        "## Data gathering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eROAs3aS6wHc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://github.com/brendenlake/omniglot/raw/master/python/images_background.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d78ikRIV-Y_N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://github.com/brendenlake/omniglot/raw/master/python/images_evaluation.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sL-MdFex8spV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip images_background.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIlL1x9_-bDt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip images_evaluation.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCrqYMNr8_8o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from matplotlib import pyplot as plt\n",
        "from PIL import Image\n",
        "import multiprocessing as mp\n",
        "import pandas as pd \n",
        "import numpy as np \n",
        "import cv2\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLP_-hY8c2xp",
        "colab_type": "code",
        "outputId": "4ed3cb39-de3a-4185-8efa-a2db8ed61b99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "random_10_bengali = os.listdir('/content/images_background/Bengali/character01/')\n",
        "random_10_bengali"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['0132_16.png',\n",
              " '0132_01.png',\n",
              " '0132_12.png',\n",
              " '0132_17.png',\n",
              " '0132_08.png',\n",
              " '0132_15.png',\n",
              " '0132_18.png',\n",
              " '0132_19.png',\n",
              " '0132_04.png',\n",
              " '0132_06.png',\n",
              " '0132_07.png',\n",
              " '0132_11.png',\n",
              " '0132_03.png',\n",
              " '0132_10.png',\n",
              " '0132_09.png',\n",
              " '0132_14.png',\n",
              " '0132_05.png',\n",
              " '0132_13.png',\n",
              " '0132_20.png',\n",
              " '0132_02.png']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JBl4HuTTceiz",
        "colab_type": "code",
        "outputId": "34a54a61-5272-4665-9c89-7ccf296bb571",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "Image.open('/content/images_background/Bengali/character01/0132_01.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAAA9ElEQVR4nN2UMU7EMBBF3ziRktIH\noMhBKLgSJ8B7A44UbsARgkRBmZW22GKTobDXzpAN0grBSrh7+t/f8y3LoizWzmHWP0b1l9VJRETc\nXsRb9XjNQS0OKlVV1UZ1vH3fH+HpWzVcEzVvq+J5KDhxWKoHa35lRgv2KF3IGJ76aK7ThG/4Ym5M\nlKaAs9quZ24NhqLOxNyNRj7mFrXdNlfD+lyLL51VPyI+G7M+AnDMezuAMeMIMMTrqZOvD3sAJ/0J\nmHbnvvcT8J6SayqEfHnrRkNqnNAvOrnou8tRNItv5/bP++9wsDgarK3ZeYPyZe/vDXkBPwHQkznd\ntAZ8bwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=1 size=105x105 at 0x7F90E3252390>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HCBC0COemXH",
        "colab_type": "code",
        "outputId": "dd21b59b-955b-4479-9b0d-8d0559e37ac4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "Image.open('/content/images_background/Bengali/character01/0132_10.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAABP0lEQVR4nO3TsU6EQBCA4X8BA4WJ\n2F3JI1jaeFL4UMbm9t7DwkdZLz6Aj4DmiisxsSAGGItdYDbxEqPtbcXHDLMzCxhBrW1CtE78iR/G\nXAZujTEltDUkMNiQ4flypJSFLFym/dt7iG5EpKe58TS+lAnPrn/T5O6PEw11xH4X8SC15hONpqVV\nHLCdYp/e9zo5y0fFbMUm7qq2EauoSUqnm6wLH/UnechDWwn0QJUdm3ckdYp+tRMFrqebvkTuZEm2\nUSkA1EQO+IyjbTmziTbqWoBujnZA2U8seqAYJp75TWxgov7ZBHAWDtXMCmhWSxsO3J3PFpHuQoQx\nFREbmhxSMydn3TQBCZB88Xru980AI9aU6nDs4/5WzVvvcYoVuT6rFcUyAuRcIeNciue1Tl4++Pkd\nmZiqFAAPMbOYnPgvfgM1ZFx1YvSO4wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=1 size=105x105 at 0x7F90C31E7EF0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fWdY-TXeqTw",
        "colab_type": "code",
        "outputId": "4b590eed-bd73-4b46-ecaf-46a521485b65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "Image.open('/content/images_background/Bengali/character01/0132_02.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAABDklEQVR4nO2SMU7EMBREn73WkmLF\nrqjoCDfgCOEmHIGS0tyEY1B6b+KCA3glCiMl+RQ4ji1BWAQUSLh7mu/R/LGVUJx7TXX+Gu4NGF6a\nH7DaCmhORERE+ovwa5k/xINqSww8legZS3RIiRbsjLISW6ij4bYcznVpgL6OEeM3Vrgj1KovUm0t\nrlZtjbWzqL5ST2mPiTHU2FuAmIddfdcvWZkAuR+d9CEPR0Dl2lPax0kdAEn+GqwFUjsaugcYN3QJ\nW4Cz7Hwe5mfRYIB4mVE/T0ZgQI3gu2uZVhBwb8YYwCqQfV7wBlSx7w7WBTbz59DAGnZcZSslkBrM\nXTXvV3cc+iXVfMVKtxUqtzT8j5/iK4N3Rb2z1Ok9AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=1 size=105x105 at 0x7F90C3177128>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rxb328gS9gJp",
        "colab_type": "text"
      },
      "source": [
        "## Data reading and augmentation\n",
        "The Omniglot data set is designed for developing more human-like learning algorithms. It contains 1623 different handwritten characters from 50 different alphabets. Then to increase the number of classes, all the images are rotated by 90, 180 and 270 degrees and each rotation resulted in one more class. Hence the total count of classes reached to 6492(1623 * 4) classes. We split images of 4200 classes to training data and the rest went to test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdMcXOli9bFl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dir = os.listdir('images_background/')\n",
        "datax = np.array([])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XEssKQ6f9z73",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def image_rotate(img, angle):\n",
        "    \"\"\"\n",
        "    Image rotation at certain angle. It is used for data augmentation \n",
        "    \"\"\"\n",
        "    rows,cols, _ = img.shape\n",
        "    M = cv2.getRotationMatrix2D((cols/2 ,rows/2),angle,1)\n",
        "    dst = cv2.warpAffine(img,M,(cols,rows))\n",
        "    return np.expand_dims(dst, 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h-b0wy9J953l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_alphabets(alphabet_directory, directory):\n",
        "    \"\"\"\n",
        "    Reads all the characters from alphabet_directory and augment each image with 90, 180, 270 degrees of rotation.\n",
        "    \"\"\"\n",
        "    datax = None\n",
        "    datay = []\n",
        "    characters = os.listdir(alphabet_directory)\n",
        "    for character in characters:\n",
        "        images = os.listdir(alphabet_directory + character + '/')\n",
        "        for img in images:\n",
        "            image = cv2.resize(cv2.imread(alphabet_directory + character + '/' + img), (28,28))\n",
        "            image90 = image_rotate(image, 90)\n",
        "            image180 = image_rotate(image, 180)\n",
        "            image270 = image_rotate(image, 270)\n",
        "            image = np.expand_dims(image, 0)\n",
        "            if datax is None:\n",
        "                datax = np.vstack([image, image90, image180, image270])\n",
        "            else:\n",
        "                datax = np.vstack([datax, image, image90, image180, image270])\n",
        "            datay.append(directory + '_' + character + '_0')\n",
        "            datay.append(directory + '_' + character + '_90')\n",
        "            datay.append(directory + '_' + character + '_180')\n",
        "            datay.append(directory + '_' + character + '_270')\n",
        "    return datax, np.array(datay)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3j4URJB_98pC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_images(base_directory):\n",
        "    \"\"\"\n",
        "    Used multithreading for data reading to decrease the reading time drastically\n",
        "    \"\"\"\n",
        "    datax = None\n",
        "    datay = []\n",
        "    pool = mp.Pool(mp.cpu_count())\n",
        "    results = [pool.apply(read_alphabets, args=(base_directory + '/' + directory + '/', directory, )) for directory in os.listdir(base_directory)]\n",
        "    pool.close()\n",
        "    for result in results:\n",
        "        if datax is None:\n",
        "            datax = result[0]\n",
        "            datay = result[1]\n",
        "        else:\n",
        "            datax = np.vstack([datax, result[0]])\n",
        "            datay = np.concatenate([datay, result[1]])\n",
        "    return datax, datay"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yPpRsOd-9_A7",
        "colab_type": "code",
        "outputId": "71cc1d43-5291-41e4-d9f2-92f9f32f344d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%time trainx, trainy = read_images('images_background/')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1.08 s, sys: 613 ms, total: 1.69 s\n",
            "Wall time: 15.5 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TvFXXdoT-Ew8",
        "colab_type": "code",
        "outputId": "3d6978a8-d6b4-4c3b-b1f6-abbac8eaa202",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%time testx, testy = read_images('images_evaluation/')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 516 ms, sys: 321 ms, total: 838 ms\n",
            "Wall time: 10.5 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gR7DaBrh-uil",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "assert(trainx.shape[0] == trainy.shape[0])\n",
        "assert(testx.shape[0] == testy.shape[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "51WVpbxYi2-i",
        "colab_type": "code",
        "outputId": "a66924b9-78a2-4c54-b1fc-543bf20b104f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "trainx.shape, trainy.shape, testx.shape, testy.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((77120, 28, 28, 3), (77120,), (52720, 28, 28, 3), (52720,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TnIaQ2_O_B1M",
        "colab_type": "text"
      },
      "source": [
        "## Building the network in TensorFlow 2.0 using `tf.keras`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tgJ2u9H-yDt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorflow-gpu==2.0.0-beta1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eu8Z5086_Low",
        "colab_type": "code",
        "outputId": "2ee996d7-7a24-4106-b195-c5a4d409e77c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "print(tf.__version__)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.0.0-beta1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GV02b54s_qEs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
        "from sklearn.metrics import pairwise_distances\n",
        "from tensorflow import keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38oGcl3hpj7-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "conv_block = keras.models.Sequential([\n",
        "    keras.layers.Conv2D(64, (3, 3), input_shape=(28, 28, 3)),\n",
        "    keras.layers.BatchNormalization(),\n",
        "    keras.layers.ReLU(),\n",
        "    keras.layers.MaxPool2D(pool_size=(2,2))\n",
        "])                       "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WBJoDcx4sJIm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "flatten_layer = tf.keras.layers.Flatten()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SbfdNr-rqnOh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_embeddings(support_set):\n",
        "    x = conv_block(support_set)\n",
        "    x = conv_block()\n",
        "    x = conv_block()\n",
        "    x = conv_block()\n",
        "    return flatten_layer(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HR0E241E_rzU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(keras.Model):\n",
        "    \"\"\"\n",
        "    Image2Vector CNN which takes image of dimension (28x28x3) and return column vector length 64\n",
        "    \"\"\"\n",
        "    def sub_block(self, out_channels=64, kernel_size=(3,3)):\n",
        "        block = keras.models.Sequential([\n",
        "                    keras.layers.Conv2D(filters=out_channels, \\\n",
        "                        kernel_size=kernel_size, input_shape=(28, 28, 3)),\n",
        "                    keras.layers.BatchNormalization(),\n",
        "                    keras.layers.ReLU(),\n",
        "                    keras.layers.MaxPool2D(pool_size=(2,2))\n",
        "        ])\n",
        "        return block\n",
        "    \n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        self.convnet1 = self.sub_block()\n",
        "        self.convnet2 = self.sub_block()\n",
        "        self.convnet3 = self.sub_block()\n",
        "        self.convnet4 = self.sub_block()\n",
        "\n",
        "    def call(self, x):\n",
        "        x = self.convnet1(x)\n",
        "        x = self.convnet2(x)\n",
        "        x = self.convnet3(x)\n",
        "        x = self.convnet4(x)\n",
        "        x = keras.layers.Flatten()(x)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jfN3KEYbQuL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JEKvU-M_bN0l",
        "colab_type": "text"
      },
      "source": [
        "To be continued.\n",
        "\n",
        "https://blog.floydhub.com/n-shot-learning/"
      ]
    }
  ]
}