imagenette_with_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/pytorch/examples/blob/master/imagenet/main.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import time\n",
    "import warnings\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.parallel\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.distributed as dist\n",
    "import torch.optim\n",
    "import torch.multiprocessing as mp\n",
    "import torch.utils.data\n",
    "import torch.utils.data.distributed\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.models as models\n",
    "\n",
    "from fastai.vision.models.xresnet import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.basics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.IMAGENETTE_320)\n",
    "BATCH_SIZE = 32\n",
    "NUM_WORKERS = 8\n",
    "LR = 8e-3\n",
    "\n",
    "WEIGHT_DECAY=1e-2\n",
    "NUM_EPOCHS=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindir = os.path.join(path, 'train')\n",
    "valdir = os.path.join(path, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                 std=[0.229, 0.224, 0.225])\n",
    "\n",
    "train_dataset = datasets.ImageFolder(\n",
    "    traindir,\n",
    "    transforms.Compose([\n",
    "        transforms.Resize(128),\n",
    "        transforms.CenterCrop(128),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        normalize,\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train_dataset, batch_size=BATCH_SIZE, shuffle=True,\n",
    "    num_workers=NUM_WORKERS, pin_memory=True)\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(valdir, transforms.Compose([\n",
    "        transforms.Resize(128),\n",
    "        transforms.CenterCrop(128),\n",
    "        transforms.ToTensor(),\n",
    "        normalize,\n",
    "    ])),\n",
    "    batch_size=BATCH_SIZE, shuffle=False,\n",
    "    num_workers=NUM_WORKERS, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 3, 128, 128]), torch.float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[0].shape, batch[0].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32]), torch.float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[1].shape, batch[0].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = xse_resnext50(n_out=10, act_cls=Mish, sa=1, sym=0, pool=MaxPool)\n",
    "model.cuda();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = LabelSmoothingCrossEntropy()\n",
    "\n",
    "# optimizer = torch.optim.SGD(model.parameters(), LR,\n",
    "#                             momentum=MOMENTUM,\n",
    "#                             weight_decay=WEIGHT_DECAY)\n",
    "\n",
    "optimizer = ranger(model.parameters(), LR, mom=0.95, sqr_mom=0.99, eps=1e-6, beta=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self, name, fmt=':f'):\n",
    "        self.name = name\n",
    "        self.fmt = fmt\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "    def __str__(self):\n",
    "        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n",
    "        return fmtstr.format(**self.__dict__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(output, target, topk=(1,)):\n",
    "    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n",
    "    with torch.no_grad():\n",
    "        maxk = max(topk)\n",
    "        batch_size = target.size(0)\n",
    "\n",
    "        _, pred = output.topk(maxk, 1, True, True)\n",
    "        pred = pred.t()\n",
    "        correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
    "\n",
    "        res = []\n",
    "        for k in topk:\n",
    "            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)\n",
    "            res.append(correct_k.mul_(100.0 / batch_size))\n",
    "        return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.callback.schedule import combined_cos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeV0lEQVR4nO3de5SV9X3v8fdnZhgQRJgbFxlwBhjQiSIhIwyaGI1JFG1C7kebxtR2lUWNWW3WahNPc1ZOTtrTlaSX9HhitaT19NjVE2pzaUhKj5q0URtFGaKCiOgwXhgdYUAEAREHvueP2dg527k8w+yZZ/Z+Pq+19nL28/x+7O9PWPsz+/fbz/NTRGBmZtlTlnYBZmaWDgeAmVlGOQDMzDLKAWBmllEOADOzjKpIu4DhqK2tjYaGhrTLMDMrKlu2bNkXEXX5x4sqABoaGmhra0u7DDOzoiLp+f6OewrIzCyjHABmZhnlADAzyygHgJlZRjkAzMwyKlEASLpK0k5J7ZJu7ue8JN2SO79V0rKh+kpaKmmTpMcktUlaXpghmZlZEkMGgKRy4FZgFdAMXCepOa/ZKqAp91gD3Jag7zeB/xYRS4Gv5J6bmdkYSXIdwHKgPSI6ACStB1YDT/Zpsxq4M3rvLb1J0nRJs4GGQfoGcFau/zTgpZEPp38/27GHx3e/Olp/vJkNh8SssybRUDuZ+bVnMmvapLQryqwkATAH2N3neSewIkGbOUP0/V3gbkl/Su8nkYv7e3FJa+j9VMG8efMSlPt29z3dzd9t6vc6CDMbY/lbkFy2uI4/+sj51FdNTqegDEsSAOrnWP4uMgO1GazvbwNfiIjvS/oU8DfA+9/WOGIdsA6gpaXltHav+drq8/na6vNPp6uZFdjJk8FLB1/nuX1HefSFA9x23y4++K37+eKVi/nMygbKy/p727DRkGQRuBOY2+d5PW+frhmozWB9Pwv8IPfzP9I71WRmJa6sTNRXTebdTbV8/oom7vnCpVzUUM1Xf/wkv7P+UU6e9C6FYyVJAGwGmiQ1SqoErgU25LXZAFyf+zZQK3AwIrqG6PsS8N7cz+8DnhnhWMysCNVXTeZvb7iI379yMT/Z2sU37n4q7ZIyY8gpoIjokXQTcDdQDtwREdslrc2dvx3YCFwNtANHgRsG65v7o38L+B+SKoBj5Ob5zSx7JHHjZQvoOvg6f3VfB3OrJvNrreekXVbJUzFtCt/S0hK+G6hZ6eo5cZI1f7eFn+/cy9/8+kVcvnhG2iWVBElbIqIl/7ivBDazcaOivIz/ed07WTRzKl/83lYOHXsz7ZJKmgPAzMaVKRMr+OYnlrDv8Bv8+T1Pp11OSXMAmNm4s6R+Or+24hzufOg5nnjxYNrllCwHgJmNS7935WKqp1TyX/7pCX81dJQ4AMxsXJp2xgS+fM15PLb7Vb67+YW0yylJDgAzG7c+snQOyxuq+YufPsOxN0+kXU7JcQCY2bglid99fxPdr73B97Z0pl1OyXEAmNm4tnJBDUvnTuf2+3bRc+Jk2uWUFAeAmY1rkvjc5QvpPPA6P946aneNzyQHgJmNe1ecO4PFM6fyl/+2y98IKiAHgJmNe2Vl4sbLF/DM3sPcu2NP2uWUDAeAmRWFay6YzTk1k7n9vl1pl1IyHABmVhQqysu4fmUDj77wKju6DqVdTklwAJhZ0fjYO+dQWV7G+kd8YVghOADMrGhUTalk1QWz+OGjL/L6cV8YNlIOADMrKtctn8ehYz1s3NaVdilFL1EASLpK0k5J7ZJu7ue8JN2SO79V0rKh+kr6B0mP5R7PSXqsICMys5K2orGa+bVT+K6ngUZsyACQVA7cCqwCmoHrJDXnNVsFNOUea4DbhuobEf8pIpZGxFLg+/zHBvFmZgOSxLXL59L2/AGe2fNa2uUUtSSfAJYD7RHRERHHgfXA6rw2q4E7o9cmYLqk2Un6ShLwKeC7IxyLmWXEx5fVM6FcfPeR3WmXUtSSBMAcoO//5c7csSRtkvR9D7AnIp7p78UlrZHUJqmtu7s7QblmVupqzpzIB5tn8cNHO3nT9wc6bUkCQP0cy78We6A2SfpexyC//UfEuohoiYiWurq6QQs1s+z4yDvncODom/yifV/apRStJAHQCczt87weyL8j00BtBu0rqQL4GPAPyUs2M4NLF9UydVIFP37c3wY6XUkCYDPQJKlRUiVwLbAhr80G4Prct4FagYMR0ZWg7/uBpyLCN/o2s2GZWFHOVe+YxT3bX/ZmMadpyACIiB7gJuBuYAdwV0Rsl7RW0tpcs41AB9AOfAe4cbC+ff74a/Hir5mdpg9deDavvdHDfU97ffB0KKJ4bq3a0tISbW1taZdhZuNEz4mTLP/jn3Hxghq+/avLhu6QUZK2RERL/nFfCWxmRauivIyrL5jFz3bs5ejxnrTLKToOADMrah9acjavv3mCn+7Ym3YpRccBYGZF7aKGamadNYkfP+7tIofLAWBmRa2sTFyzZDb37ezm8BueBhoOB4CZFb0PNs/k+ImT3O9vAw2LA8DMit67zqmiavIE7n3S+wUPhwPAzIpeRXkZ7zt3Jv/61F7fG2gYHABmVhI+0DyDg6+/yebnXkm7lKLhADCzkvCepjoqK8o8DTQMDgAzKwlTJlbw7oW13PvkHorpDgdpcgCYWcn4QPNMOg+8zlMve6ewJBwAZlYyrjhvBhKeBkrIAWBmJWPG1EksnTvdAZCQA8DMSsr7z5vJthcPsufQsbRLGfccAGZWUi5b3Lt1rPcIGJoDwMxKSvPss6ibOtEBkECiAJB0laSdktol3dzPeUm6JXd+q6RlSfpK+nzu3HZJ3xz5cMws6yTx3kV1PPB0Nz2+KnhQQwaApHLgVmAV0AxcJ6k5r9kqoCn3WAPcNlRfSZcDq4ElEfEO4E8LMSAzs8sW13HoWA+Pd76adinjWpJPAMuB9ojoiIjjwHp637j7Wg3cGb02AdMlzR6i728DX4+INwAiwrs5mFlBvHthLWWCn+/0NNBgkgTAHGB3n+eduWNJ2gzWdxHwHkkPS7pP0kX9vbikNZLaJLV1d/sv08yGNn1yJe+cV+V1gCEkCQD1cyz/OuuB2gzWtwKoAlqB3wfukvS29hGxLiJaIqKlrq4uQblmZvDeRXVs7TzIvsNvpF3KuJUkADqBuX2e1wP5e68N1Gawvp3AD3LTRo8AJ4Ha5KWbmQ3s1NdBH3jGnwIGkiQANgNNkholVQLXAhvy2mwArs99G6gVOBgRXUP0/SfgfQCSFgGVwL6RDsjMDOD8s6dRM6XS6wCDqBiqQUT0SLoJuBsoB+6IiO2S1ubO3w5sBK4G2oGjwA2D9c390XcAd0h6AjgOfDZ8Cz8zK5CyMnHpojp+vnMvJ04G5WX9zUhn25ABABARG+l9k+977PY+PwfwuaR9c8ePA782nGLNzIbj0kW1/PDRF3nypUNcUD8t7XLGHV8JbGYl65KFvcuK/97u2eX+OADMrGTNmDqJxTOn8gsHQL8cAGZW0i5ZWMsjz73CsTdPpF3KuOMAMLOS9p6mWo73nGTL8wfSLmXccQCYWUlb3lhNRZm8DtAPB4CZlbQpEytYNq/K6wD9cACYWcm7ZGEt2148yKtHj6ddyrjiADCzkvfuphoi4KFd+9MuZVxxAJhZyVtSP50zJ1Z4HSCPA8DMSt6E8jJa51d7HSCPA8DMMuGShbU8t/8ou185mnYp44YDwMwy4eIFvbeFeKjD6wCnOADMLBMWzTyTmimVbPJC8FscAGaWCZJoXVDDQx378Z3nezkAzCwzVs6voevgMZ7f73UASBgAkq6StFNSu6Sb+zkvSbfkzm+VtGyovpK+KulFSY/lHlcXZkhmZv1buaAG8DrAKUMGgKRy4FZgFdAMXCepOa/ZKqAp91gD3Jaw77ciYmnu8bZNY8zMCml+7RRmTJ3Ig14HAJJ9AlgOtEdER24Xr/XA6rw2q4E7cxu8bwKmS5qdsK+Z2ZiQxMoFNTy0y+sAkCwA5gC7+zzvzB1L0maovjflpozukFTV34tLWiOpTVJbd7c3dzazkVk5v4Z9h99gV/fhtEtJXZIA6G8n5fzoHKjNYH1vAxYAS4Eu4M/6e/GIWBcRLRHRUldXl6BcM7OBvbUO4GmgRAHQCczt87weeClhmwH7RsSeiDgRESeB79A7XWRmNqrmVU/m7GmTvBBMsgDYDDRJapRUCVwLbMhrswG4PvdtoFbgYER0DdY3t0ZwykeBJ0Y4FjOzIb11PcCu/Zw8me11gCEDICJ6gJuAu4EdwF0RsV3SWklrc802Ah1AO72/zd84WN9cn29K2iZpK3A58IXCDcvMbGAr59dw4Oib7NzzWtqlpKoiSaPcVzQ35h27vc/PAXwuad/c8c8Mq1IzswI5tQ7wcMd+zpt9VsrVpMdXAptZ5tRXTWbO9DPY1PFK2qWkygFgZpnUOr+GR557JdPrAA4AM8ukFfOreeXIcZ7Zm93rARwAZpZJrY25dYBns/t1UAeAmWXS3OozOHvaJB7O8DqAA8DMMkkSK+bX8PCz2b0vkAPAzDKrdX41+w4fz+x9gRwAZpZZKxpP7Q+QzWkgB4CZZdY5NZOZddYkHs7ofYEcAGaWWb3rANU8/OwrmVwHcACYWaataKyh+7U36Nh3JO1SxpwDwMwyrXV+NUAmvw7qADCzTGusnULd1ImZvCDMAWBmmSaJ5Y3VPNyRvXUAB4CZZV5rYzUvHzrG7ldeT7uUMeUAMLPMWzG/93qATRmbBkoUAJKukrRTUrukm/s5L0m35M5vlbRsGH1/T1JIqh3ZUMzMTk/TjDOpnlKZuYXgIQNAUjlwK7AKaAauk9Sc12wV0JR7rAFuS9JX0lzgA8ALIx6JmdlpksRFDVWZWwhO8glgOdAeER0RcRxYD6zOa7MauDN6bQKm5zZ9H6rvt4AvAtlaeTGzcWdFYw2dB17nxVezsw6QJADmALv7PO/MHUvSZsC+kj4MvBgRjw/24pLWSGqT1Nbd3Z2gXDOz4VuRux7gkQx9CkgSAOrnWP5v7AO16fe4pMnAl4GvDPXiEbEuIloioqWurm7IYs3MTse5s87irEkVmVoHSBIAncDcPs/rgZcSthno+AKgEXhc0nO547+UNGs4xZuZFUp5We56gGcdAH1tBpokNUqqBK4FNuS12QBcn/s2UCtwMCK6BuobEdsiYkZENEREA71BsSwiXi7UwMzMhmt5YzXP7jvC3kPH0i5lTAwZABHRA9wE3A3sAO6KiO2S1kpam2u2EegA2oHvADcO1rfgozAzK4AVb+0TnI1PARVJGkXERnrf5Pseu73PzwF8Lmnffto0JKnDzGw0vePsszhzYgUPP7ufD114dtrljDpfCWxmllNRXsa7zqliU0YWgh0AZmZ9rJhfTfvew+w7/EbapYw6B4CZWR+tufsCPZKBdQAHgJlZHxfMmcbkyvJM7BPsADAz62NCbh0gC98EcgCYmeVZ0VjNUy+/xoEjx9MuZVQ5AMzM8pzaH6DUPwU4AMzM8iypn8bEirKSvz20A8DMLM/EinKWzasq+RvDOQDMzPqxYn41O14+xMGjb6ZdyqhxAJiZ9aN1fg0RsPm50v0U4AAwM+vH0rnTqawoY1MJXw/gADAz68ekCeUsmzedhxwAZmbZ0zq/hie7SncdwAFgZjaAlbl1gFL9OqgDwMxsAEvnTWdiRVnJTgMlCgBJV0naKald0s39nJekW3Lnt0paNlRfSX+Ya/uYpHsklf7uC2ZWVCZWlJf0/gBDBoCkcuBWYBXQDFwnqTmv2SqgKfdYA9yWoO+fRMSSiFgK/AT4yohHY2ZWYCvn17Cj61BJ3hcoySeA5UB7RHRExHFgPbA6r81q4M7otQmYLmn2YH0j4lCf/lOAGOFYzMwKbuWCU/cFKr1poCQBMAfY3ed5Z+5YkjaD9pX03yXtBj7NAJ8AJK2R1Caprbu7O0G5ZmaFs6R+OmdMKC/JaaAkAaB+juX/tj5Qm0H7RsSXI2Iu8PfATf29eESsi4iWiGipq6tLUK6ZWeFUVpTR0lDFQ7uy+QmgE5jb53k98FLCNkn6Avwf4OMJajEzG3Ot82vYuec19pfYPsFJAmAz0CSpUVIlcC2wIa/NBuD63LeBWoGDEdE1WF9JTX36fxh4aoRjMTMbFa0luj9AxVANIqJH0k3A3UA5cEdEbJe0Nnf+dmAjcDXQDhwFbhisb+6P/rqkxcBJ4HlgbUFHZmZWIEvqe/cJfnDXPq6+YHba5RTMkAEAEBEb6X2T73vs9j4/B/C5pH1zxz3lY2ZFYUJ5GSsaq3mwvbTWAXwlsJlZApcsrKVj3xG6Dr6edikF4wAwM0vg4gW1APyihD4FOADMzBI4d9ZUqqdU8mD7vrRLKRgHgJlZAmVlYuWCGn6xax+9y57FzwFgZpbQJQtq2XPoDXZ1H0m7lIJwAJiZJfTuhafWAUpjGsgBYGaW0LyaydRXneEAMDPLoksW1LKpYz8nThb/OoADwMxsGC5eWMOhYz088eLBtEsZMQeAmdkwvHU9wK7inwZyAJiZDUPd1ImcO2sqDzztADAzy5xLF9XR9vwrHHmjJ+1SRsQBYGY2TJc21fHmiWBTR3HfFsIBYGY2TC0NVZwxoZz7ny7ubWodAGZmwzRpQjmt86u5/5niXgdIFACSrpK0U1K7pJv7OS9Jt+TOb5W0bKi+kv5E0lO59j+UNL0gIzIzGwOXLqrj2X1HeGH/0bRLOW1DBoCkcuBWYBXQDFwnqTmv2SqgKfdYA9yWoO+9wPkRsQR4GvjPIx6NmdkYuXRRHQD3PVO800BJPgEsB9ojoiMijgPrgdV5bVYDd0avTcB0SbMH6xsR90TEqSX0TfRuGG9mVhTm106hvuqMol4HSBIAc4DdfZ535o4laZOkL8BvAP/S34tLWiOpTVJbd3fx/o82s9IiiUsX1fHQrv28eeJk2uWcliQBoH6O5d8EY6A2Q/aV9GWgB/j7/l48ItZFREtEtNTV1SUo18xsbFzaVMfhN3r45fMH0i7ltCQJgE5gbp/n9cBLCdsM2lfSZ4FfAT4dpbLDgpllxsULaygvE/cV6TRQkgDYDDRJapRUCVwLbMhrswG4PvdtoFbgYER0DdZX0lXAl4APR0TxLqObWWadNWkC7zqnin/bWaIBkFuovQm4G9gB3BUR2yWtlbQ212wj0AG0A98Bbhysb67Pt4GpwL2SHpN0e+GGZWY2Nq44dwY7ug7x4quvp13KsKmYZl5aWlqira0t7TLMzN7Svvcw7//z+/jD1e/gMysb0i6nX5K2RERL/nFfCWxmNgIL6qbQUDOZn+7Ym3Ypw+YAMDMbAUlccd5MHtq1v+juDuoAMDMbofefN5PjJ07yQJHdG8gBYGY2Qi0NVZw1qYKf7tiTdinD4gAwMxuhCeVlXLZ4Bv/21N6i2izeAWBmVgBXnDeD/UeO89juV9MuJTEHgJlZAVy2aAblZeJnRTQN5AAwMyuAaZMnsLyhmnuedACYmWXO1RfMon3vYZ7e81rapSTiADAzK5Arz5+FBP+8tSvtUhJxAJiZFciMqZNY0VjNP29zAJiZZc41F8wummkgB4CZWQGdmgb6SRFMAzkAzMwK6NQ00MZtXYz3uy07AMzMCuw/poEOp13KoBwAZmYFduX5sygT434xOFEASLpK0k5J7ZJu7ue8JN2SO79V0rKh+kr6pKTtkk5KettGBWZmxWrG1Eksb6zmJ4+/NK6ngYYMAEnlwK3AKqAZuE5Sc16zVUBT7rEGuC1B3yeAjwH3j3wYZmbjy8eW1dOx7wi/fOFA2qUMKMkngOVAe0R0RMRxYD2wOq/NauDO6LUJmC5p9mB9I2JHROws2EjMzMaRay6YzeTKcu7a3Jl2KQNKEgBzgN19nnfmjiVpk6TvoCStkdQmqa27u3s4Xc3MUjNlYgXXXDCbn2x9iaPHx+dOYUkCQP0cy5/UGqhNkr6Dioh1EdESES11dXXD6WpmlqpPXTSXI8dPsHHby2mX0q8kAdAJzO3zvB54KWGbJH3NzEpSyzlVNNZO4R/bdg/dOAVJAmAz0CSpUVIlcC2wIa/NBuD63LeBWoGDEdGVsK+ZWUmSxCfeVc/Dz77Cc/uOpF3O2wwZABHRA9wE3A3sAO6KiO2S1kpam2u2EegA2oHvADcO1hdA0kcldQIrgX+WdHdBR2ZmNg58fFk9ZYLvbRl/i8Eaz99RzdfS0hJtbW1pl2FmNiw3/K9H2NH1Gg986XImlI/99beStkTE26638pXAZmaj7PqVDbx86Ni42yfAAWBmNsouW1xH04wz+av7O8bVlcEOADOzUSaJ37p0Pju6DvGL9v1pl/MWB4CZ2RhYvfRs6qZOZN0DHWmX8hYHgJnZGJhYUc6vX9zA/U93s6PrUNrlAA4AM7Mx8+kV85hcWc5fP/Bs2qUADgAzszEzfXIln2qZy48ee5GO7vQ3i3EAmJmNoc9dvpBJE8r544070i7FAWBmNpbqpk7kpvct5Kc79vLAM+ne4dgBYGY2xm64pIF51ZP5o5/soOfEydTqcACYmY2xiRXl/MHV57Jzz2us35zenUIdAGZmKbjyHbNY0VjNn92zk72HjqVSgwPAzCwFkvijj5zPsTdPcuPf/5LjPWM/FeQAMDNLSdPMqXzjE0toe/5AKt8KqhjzVzQzs7d8+MKzeXz3q/zNvz/LhXOn8dF31o/ZazsAzMxSdvOqc9n24kFu/v42KsrK+NCFZ4/J6yaaApJ0laSdktol3dzPeUm6JXd+q6RlQ/WVVC3pXknP5P5bVZghmZkVlwnlZfzlp5dx/pxpfP67j/LVDdvHZE1gyACQVA7cCqwCmoHrJDXnNVsFNOUea4DbEvS9GfhZRDQBP8s9NzPLpNozJ7J+TSu/cUkjf/vgc3zyrx7iHza/wO5Xjo7aayaZAloOtEdEB4Ck9cBq4Mk+bVYDd0bvTgebJE2XNBtoGKTvauCyXP//Dfwc+NIIx2NmVrQmlJfxlQ81s+yc6Xztx0/ype9vA2Bu9Rl84+NLuHhBbUFfL0kAzAH6XqnQCaxI0GbOEH1nRkQXQER0SZrR34tLWkPvpwrmzZuXoFwzs+L2K0vO5poLZtO+9zC/aN/Hg7v2M+usSQV/nSQBoH6O5e9pNlCbJH0HFRHrgHXQuyn8cPqamRUrSTTNnErTzKn8+iWNo/IaSRaBO4G5fZ7XAy8lbDNY3z25aSJy/92bvGwzMxupJAGwGWiS1CipErgW2JDXZgNwfe7bQK3Awdz0zmB9NwCfzf38WeBHIxyLmZkNw5BTQBHRI+km4G6gHLgjIrZLWps7fzuwEbgaaAeOAjcM1jf3R38duEvSbwIvAJ8s6MjMzGxQ6v3iTnFoaWmJtra2tMswMysqkrZEREv+cd8LyMwsoxwAZmYZ5QAwM8soB4CZWUYV1SKwpG7g+dPsXgvsK2A5xcBjzgaPORtGMuZzIqIu/2BRBcBISGrrbxW8lHnM2eAxZ8NojNlTQGZmGeUAMDPLqCwFwLq0C0iBx5wNHnM2FHzMmVkDMDOz/1+WPgGYmVkfDgAzs4wquQAYyQb2xSrBmD+dG+tWSQ9KujCNOgtpqDH3aXeRpBOSPjGW9RVakvFKukzSY5K2S7pvrGsstAT/rqdJ+rGkx3NjviGNOgtJ0h2S9kp6YoDzhX3/ioiSedB7y+ldwHygEngcaM5rczXwL/TuVtYKPJx23WMw5ouBqtzPq7Iw5j7t/pXe25V/Iu26R/nveDq9e23Pyz2fkXbdYzDmPwC+kfu5DngFqEy79hGO+1JgGfDEAOcL+v5Vap8A3trAPiKOA6c2oe/rrQ3sI2ITcGoD+2I15Jgj4sGIOJB7uonendmKWZK/Z4DPA9+n+HebSzLeXwV+EBEvAEREFsYcwFRJAs6kNwB6xrbMwoqI++kdx0AK+v5VagEw0Ob0w21TTIY7nt+k9zeIYjbkmCXNAT4K3D6GdY2WJH/Hi4AqST+XtEXS9WNW3ehIMuZvA+fRu83sNuB3IuLk2JSXmoK+fyXZFL6YjGQD+2KVeDySLqc3AN49qhWNviRj/gvgSxFxovcXxKKWZLwVwLuAK4AzgIckbYqIp0e7uFGSZMxXAo8B7wMWAPdKeiAiDo1ybWkq6PtXqQXASDawL1aJxiNpCfDXwKqI2D9GtY2WJGNuAdbn3vxrgasl9UTEP41JhYWV9N/1vog4AhyRdD9wIVCsAZBkzDcAX4/eyfF2Sc8C5wKPjE2JqSjo+1epTQGNZAP7YjXkmCXNA34AfKaIfyPsa8gxR0RjRDRERAPwPeDGIn3zh2T/rn8EvEdShaTJwApgxxjXWUhJxvwCvZ94kDQTWAx0jGmVY6+g718l9QkgRrCBfbFKOOavADXAX+Z+I+6JIr6TYsIxl4wk442IHZL+L7AVOAn8dUT0+1XCYpDw7/gPgb+VtI3eqZEvRURR3yJa0neBy4BaSZ3AfwUmwOi8f/lWEGZmGVVqU0BmZpaQA8DMLKMcAGZmGeUAMDPLKAeAmVlGOQDMzDLKAWBmllH/D6lzkqJHaqUeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.linspace(0.,1,100)\n",
    "f = combined_cos(0.75, LR, LR, LR/100000)\n",
    "plt.plot(p, [f(o) for o in p]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdZ0lEQVR4nO3df5DU9Z3n8edrZhhAfiMD4kwrGEcNUcChIWQ3MWqyOTCJxB8MeqkzlU0dx7ru7m1d7uJdqnJ3u3VVuSRXe2etp4d71q13exFQoxhJiGdMzG5CMg0CShCdEH8MIAz+QMUoDrzvj26yk3Zm+gvTM9/+8XpUdTn97c+Hfn+E6td8v9/+vr+KCMzMrP40pF2AmZmlwwFgZlanHABmZnXKAWBmVqccAGZmdaop7QJOxYwZM2LOnDlpl2FmVlW2bt16OCJairdXVQDMmTOHXC6XdhlmZlVF0gsDbfchIDOzOuUAMDOrUw4AM7M65QAwM6tTDgAzszqVKAAkLZO0R1K3pFsHeF2Sbiu8vlNSR6m5khZK2iJpu6ScpCXlWZKZmSVRMgAkNQK3A8uBecCNkuYVDVsOtBceq4E7Esz9BvAfI2Ih8LXCczMzGyVJrgNYAnRHxF4ASfcCK4Bf9huzArgn8r2lt0iaKmk2MGeIuQFMLsyfAuwf/nIG9tjug+x46fWR+uPN7BRNHNfEF39/LmMafRQ6TUkCoBV4qd/zHuDDCca0lpj7L4HNkr5Ffk/k9wZ6c0mrye9VcM455yQo9/1+/Gwv/3vLgNdBmNkoO3kLkrOmjOfqBWenW0ydSxIAGmBb8V1kBhsz1Nw/Av48Iu6X1An8T+CT7xscsRZYC5DNZk/r7jV/seJi/mLFxacz1czK7MSJ4GPfeJwNuZccAClLsv/VA2T6PW/j/YdrBhsz1NwvAA8Uft5A/lCTmdW4hgaxMtvG33cfpue1t9Mup64lCYAuoF3SXEnNwA3AxqIxG4GbCt8GWgociYgDJebuBz5e+PlK4LlhrsXMqsT1i9oAuG9rT8qV1LeSh4Aiok/SLcBmoBG4OyJ2SVpTeP1OYBNwFdANvA18cai5hT/6nwP/TVIT8A6F4/xmVvvapp3BR8+fwYZcD396ZTsNDQMdLbaRpmq6KXw2mw13AzWrDRt37OdPv/0k/+dLH+aj7TPSLqemSdoaEdni7f4Olpml4lPzZjFl/BjW5V4qPdhGhAPAzFIxbkwj11zayuZdL/P628fSLqcuOQDMLDUrs20c6zvBQ9tH7DpQG4IDwMxS86Gzp3Bx62TWdfkwUBocAGaWqs5shl8eeIOn9x1Ju5S64wAws1StWNBKc1MD630yeNQ5AMwsVVPOGMOyD53Fg0/u4533jqddTl1xAJhZ6lYtzvDGO31s3vVy2qXUFQeAmaXuI+edSdu08WzIuTXEaHIAmFnqGhrEykUZ/r77MC+96gZxo8UBYGYV4fpsG5IbxI0mB4CZVYTWqeP56PkzuG9rD8dPVE+PsmrmADCzitGZzbDv9d/w018dTruUuuAAMLOK8akPzWLqGWN8ZfAocQCYWcUY29TI5xa28oNdB3ntqBvEjTQHgJlVlM5shmPHT/DQ9n1pl1LzEgWApGWS9kjqlnTrAK9L0m2F13dK6ig1V9I6SdsLj+clbS/Lisysqs07ezKXtE5hXa6HarphVTUqGQCSGoHbgeXAPOBGSfOKhi0H2guP1cAdpeZGxKqIWBgRC4H7+ccbxJtZnevMtrH7wBvs2v9G2qXUtCR7AEuA7ojYGxHHgHuBFUVjVgD3RN4WYKqk2UnmShLQCXx7mGsxsxpx9cJWxjY1+GTwCEsSAK1A/7+FnsK2JGOSzP0YcDAinhvozSWtlpSTlOvt7U1QrplVuynjx7Ds4rN4aLsbxI2kJAGgAbYVH5gbbEySuTcyxG//EbE2IrIRkW1paRmyUDOrHauybhA30pIEQA+Q6fe8DSi+f9tgY4acK6kJuBZYl7xkM6sHS887k8z08b5PwAhKEgBdQLukuZKagRuAjUVjNgI3Fb4NtBQ4EhEHEsz9JPBMRLj5h5n9jpMN4v6h+xU3iBshJQMgIvqAW4DNwG5gfUTskrRG0prCsE3AXqAbuAu4eai5/f74G/DJXzMbxHWL8g3iNrhB3IhQNX3PNpvNRi6XS7sMMxtFN939C7oPvslPvnIljQ0DnVa0UiRtjYhs8XZfCWxmFW1VNsP+I+/wD91uEFduDgAzq2ifnDeTaWeMYZ1PBpedA8DMKtrYpkY+d2krj7pBXNk5AMys4p1sEPegG8SVlQPAzCreB2dPZn7bFNZ1veQGcWXkADCzqrAym+GZl9/k6X1uEFcuDgAzqwpXLzg73yAu92LapdQMB4CZVYUp48ew/OKzeGj7fjeIKxMHgJlVjc7FGd58p4/vP+0GceXgADCzqrF0rhvElZMDwMyqRkOD6FyU4ae/eoUXX3GDuOFyAJhZVbk+m28Qd99W7wUMlwPAzKrK7Cnjuay9hQ1bezh+wtcEDIcDwMyqzqrFGQ4ceYefPOfbxA6HA8DMqs4nPphvELch5/sEDIcDwMyqztimRq65tI0f/PJlXnWDuNOWKAAkLZO0R1K3pFsHeF2Sbiu8vlNSR5K5kv6k8NouSd8Y/nLMrF50Lm7jvePBg0+6QdzpKhkAkhqB24HlwDzgRknzioYtB9oLj9XAHaXmSroCWAHMj4gPAd8qx4LMrD5cdNZkFrRNYX3ODeJOV5I9gCVAd0TsjYhjwL3kP7j7WwHcE3lbgKmSZpeY+0fA1yPiXYCIOFSG9ZhZHTnZIO6pfUfSLqUqJQmAVqD/F257CtuSjBlq7gXAxyT9XNKPJS0e6M0lrZaUk5Tr7fUZfzP7R1cvLDSI6/I1AacjSQAMdBfm4v2twcYMNbcJmAYsBf41sF7S+8ZHxNqIyEZEtqWlJUG5ZlYvJo8bw1WXzGbj9v385pgbxJ2qJAHQA2T6PW8D9iccM9TcHuCBwmGjXwAngBnJSzczy98t7M13+/j+rgNpl1J1kgRAF9Auaa6kZuAGYGPRmI3ATYVvAy0FjkTEgRJzHwSuBJB0AdAMHB7ugsysvnx47nTOmX4G67t8TcCpKhkAEdEH3AJsBnYD6yNil6Q1ktYUhm0C9gLdwF3AzUPNLcy5GzhP0tPkTw5/IXwq38xOUUOD6My28bO9r/DCK0fTLqeqqJo+c7PZbORyubTLMLMKc+DIb/j9r/+QP77ifP7Vpy5Mu5yKI2lrRGSLt/tKYDOrerOnjOeyC1q4zw3iTokDwMxqwqpsvkHcE24Ql5gDwMxqwic+OIvpE5rZ4LuFJeYAMLOa0NzUwDWXtvLoLw/yylvvpl1OVXAAmFnN6Mxm8g3ithdfqmQDcQCYWc248KxJLMhMZX2XG8Ql4QAws5rSmW1jz8E32dnjBnGlOADMrKZ8dsHZjBvTwDqfDC7JAWBmNWXyuDFcdfFsHnaDuJIcAGZWczoX5xvEfe9pN4gbigPAzGrOh+dOZ86ZZ7Deh4GG5AAws5ojiZXZDFv2vuoGcUNwAJhZTbquo40GwYac20QPxgFgZjXprCnj+LgbxA3JAWBmNWvV4gwvv/EOTzzrBnEDSRQAkpZJ2iOpW9KtA7wuSbcVXt8pqaPUXEn/QdI+SdsLj6vKsyQzs7wrL5rFmROafTJ4ECUDQFIjcDuwHJgH3ChpXtGw5UB74bEauCPh3L+KiIWFx6bhLsbMrL+TDeL+3243iBtIkj2AJUB3ROyNiGPkb9+4omjMCuCewg3etwBTJc1OONfMbMR0Ls43iPvOk/vSLqXiJAmAVqD//lNPYVuSMaXm3lI4ZHS3pGkDvbmk1ZJyknK9vT6OZ2an5oJZk1iYmcr6nBvEFUsSABpgW/H/xcHGDDX3DuADwELgAPBfBnrziFgbEdmIyLa0tCQo18zsd3VmMzx78C12uEHc70gSAD1Apt/zNqC42fZgYwadGxEHI+J4RJwA7iJ/uMjMrOw+u2B2vkFcl08G95ckALqAdklzJTUDNwAbi8ZsBG4qfBtoKXAkIg4MNbdwjuCka4Cnh7kWM7MBTRo3hqsumc3DO9wgrr+SARARfcAtwGZgN7A+InZJWiNpTWHYJmAv0E3+t/mbh5pbmPMNSU9J2glcAfx5+ZZlZva7VmUzvPVuH5uecoO4k1RNJ0Wy2Wzkcrm0yzCzKhQRXPGtHzFr8jjW/YuPpF3OqJK0NSKyxdt9JbCZ1YWTDeJ+/utXef6wG8SBA8DM6sj1i/IN4nxlcJ4DwMzqxqzJ47j8wpncv62HvuMn0i4ndQ4AM6srndkMB994lyee84WlDgAzqytXXjQz3yCuy/cJcACYWV1pbmrg2o58g7jDdd4gzgFgZnWnM5uh70TwYJ03iHMAmFndaZ81iUvPmcq6rvpuEOcAMLO61JnN8Nyht9j+0utpl5IaB4CZ1aXPzJ/N+DGNdX1NgAPAzOrSpHFj+PT82Ty84wBvH+tLu5xUOADMrG51/rZB3Mtpl5IKB4CZ1a3Fc6Yxd8aEuj0M5AAws7qVbxDXxi9+/Sq/rsMGcQ4AM6tr13e00digutwLcACYWV2bOXkcl1/Qwv1b669BXKIAkLRM0h5J3ZJuHeB1Sbqt8PpOSR2nMPfLkkLSjOEtxczs9HQuznDozXf58bP11SCuZABIagRuB5YD84AbJc0rGrYcaC88VgN3JJkrKQP8AfDisFdiZnaarrxoJjMmNtfdYaAkewBLgO6I2BsRx4B7gRVFY1YA90TeFmBq4abvpeb+FfBvgPq9FtvMUjemsYFrO9p4bPchet+snwZxSQKgFegfiz2FbUnGDDpX0tXAvojYMdSbS1otKScp19tbX7tnZjZ6OrNtddcgLkkAaIBtxb+xDzZmwO2SzgC+Cnyt1JtHxNqIyEZEtqWlpWSxZman4/yZk+g4ZyrrcvXTIC5JAPQAmX7P24D9CccMtv0DwFxgh6TnC9u3STrrVIo3MyunVYszdB96iyfrpEFckgDoAtolzZXUDNwAbCwasxG4qfBtoKXAkYg4MNjciHgqImZGxJyImEM+KDoioj6vxzazivDp+WdzRnMj67vq42RwyQCIiD7gFmAzsBtYHxG7JK2RtKYwbBOwF+gG7gJuHmpu2VdhZlYGE8c28elLZvPwjv110SBO1XSsK5vNRi6XS7sMM6thXc+/yso7f8Y3r5/Pymym9IQqIGlrRGSLt/tKYDOzfrLnTuO8OmkQ5wAwM+sn3yAuQ9fzr7G39620yxlRDgAzsyLXdbQWGsT1pF3KiHIAmJkVmTl5HFdc2ML922q7QZwDwMxsAJ3ZDL1vvsuP9tRuBwIHgJnZAK64aCYzJo6t6ZPBDgAzswGMaWzguo5WfvhM7TaIcwCYmQ1iZTZD34ngO0/W5slgB4CZ2SDOnzmRRedOY11XbTaIcwCYmQ1hVTbDr3qPsu3F19MupewcAGZmQ7hq/uyabRDnADAzG8LEsU18Zv5svrtzP0ffra0GcQ4AM7MSOrMZjh47ziNPHUi7lLJyAJiZlbDo3Gmc1zKh5g4DOQDMzEqQRGc2Q+6F1/hVDTWIcwCYmSVw7W8bxNXOXkCiAJC0TNIeSd2Sbh3gdUm6rfD6TkkdpeZK+svC2O2SfiDp7PIsycys/GZOGscVF87k/q37eK9GGsSVDABJjcDtwHJgHnCjpHlFw5YD7YXHauCOBHO/GRHzI2Ih8F3ga8NejZnZCFq1OMPht2qnQVySPYAlQHdE7I2IY8C9wIqiMSuAeyJvCzBV0uyh5kbEG/3mTwBq7zI7M6spl1/YUlMN4pIEQCvQf7U9hW1Jxgw5V9J/kvQS8HkG2QOQtFpSTlKut7c2UtfMqtOYxgauW5RvEHfozXfSLmfYkgSABthW/Nv6YGOGnBsRX42IDPB3wC0DvXlErI2IbERkW1paEpRrZjZyOrMZjp8IvrNtX9qlDFuSAOgBMv2etwH7E45JMhfg/wLXJajFzCxVH2iZSPbcaazLVX+DuCQB0AW0S5orqRm4AdhYNGYjcFPh20BLgSMRcWCouZLa+82/GnhmmGsxMxsVnYsz7O09yrYXX0u7lGEpGQAR0Uf+8MxmYDewPiJ2SVojaU1h2CZgL9AN3AXcPNTcwpyvS3pa0k7gU8CflW9ZZmYj59OXzGZCcyPrqvzKYFXTLkw2m41cLpd2GWZmfOW+nTy8cz9dX/0kE8Y2pV3OkCRtjYhs8XZfCWxmdho6F7fx9rHjPLKzehvEOQDMzE5DxznT+EDLBNZV8TUBDgAzs9NwskHc1hdeo/tQdTaIcwCYmZ2mazvaaGwQG6p0L8ABYGZ2mlomjeXKi2Zy/7bqbBDnADAzG4ZV2XyDuMefOZR2KafMAWBmNgyXX9hCy6SxrM/1pF3KKXMAmJkNQ1NjA9d1tPH4nkMceqO6GsQ5AMzMhqkz28bxE8EDT1ZXgzgHgJnZMJ3XMpHFc6axvqu6GsQ5AMzMyqAzm2Hv4aNsfaF6GsQ5AMzMyuCqKmwQ5wAwMyuDCWOb+OyCs3nkqQO89W5f2uUk4gAwMyuTldlMoUHcQPe9qjwOADOzMuk4Zyrnz5xYNYeBEgWApGWS9kjqlnTrAK9L0m2F13dK6ig1V9I3JT1TGP8dSVPLsiIzs5TkG8S1se3F1+k+9Gba5ZRUMgAkNQK3A8uBecCNkuYVDVsOtBceq4E7Esx9FLg4IuYDzwL/dtirMTNL2TWXttHUoKq4MjjJHsASoDsi9kbEMeBeYEXRmBXAPZG3BZgqafZQcyPiB4VbRgJsIX/DeDOzqnayQdwD23oqvkFckgBoBfof0OopbEsyJslcgD8EvjfQm0taLSknKdfb25ugXDOzdK1anOHwW8f4YYU3iEsSABpgW/GlboONKTlX0leBPuDvBnrziFgbEdmIyLa0tCQo18wsXR+/oIWZk8ZW/H0CkgRAD5Dp97wNKP6O02Bjhpwr6QvAZ4DPRzVdP21mNoSmxgauW9TG43t6K7pBXJIA6ALaJc2V1AzcAGwsGrMRuKnwbaClwJGIODDUXEnLgK8AV0fE22Vaj5lZRejMZjh+Irh/W+U2iCsZAIUTtbcAm4HdwPqI2CVpjaQ1hWGbgL1AN3AXcPNQcwtz/hqYBDwqabukO8u3LDOzdM2dMYElc6azIVe5DeKakgyKiE3kP+T7b7uz388B/HHSuYXt559SpWZmVaZzcYYvb9hB7oXXWDxnetrlvI+vBDYzGyFXXXIWE8c2VeyVwQ4AM7MRckZzE59dMJtHdh7gzXfeS7uc93EAmJmNoJXZDL957ziP7DyQdinv4wAwMxtBl2am0j5zIusq8JoAB4CZ2QjKN4jL8OSLr/PcwcpqEOcAMDMbYdd0tBYaxFXWXoADwMxshM2YOJZPfnAWD2zbV1EN4hwAZmajoHNxG68cPcZjuyunQZwDwMxsFFzW3sKsyZXVIM4BYGY2CpoaG7iuo43H9xziYIU0iHMAmJmNks5shhMB92+rjLuFOQDMzEbJnBkTWDJ3OhtyPRXRIM4BYGY2ilZlM/z68FG6nn8t7VIcAGZmo2l5BTWIcwCYmY2ifIO4s9n0VPoN4hwAZmajrDPbxm/eO853U24QlygAJC2TtEdSt6RbB3hdkm4rvL5TUkepuZJWStol6YSkbHmWY2ZW+RZmpnLBrImpHwYqGQCSGoHbgeXAPOBGSfOKhi0H2guP1cAdCeY+DVwLPDH8ZZiZVY+TDeK2v/Q6z6bYIC7JHsASoDsi9kbEMeBeYEXRmBXAPZG3BZgqafZQcyNid0TsKdtKzMyqyDWXtjKmUaxPcS8gSQC0Av0r7ClsSzImydwhSVotKScp19vbeypTzcwq1pmFBnHfeXIfx/rSaRCXJAA0wLbiKxgGG5Nk7pAiYm1EZCMi29LScipTzcwqWmc2wytHj/HDZw6m8v5JAqAHyPR73gbsTzgmyVwzs7p02QUtnDV5HOtz6bSGSBIAXUC7pLmSmoEbgI1FYzYCNxW+DbQUOBIRBxLONTOrS40N4rpFrfxozyFePjL6DeJKBkBE9AG3AJuB3cD6iNglaY2kNYVhm4C9QDdwF3DzUHMBJF0jqQf4CPCIpM1lXZmZWRVYuSi9BnGqhIZESWWz2cjlcmmXYWZWVqv+x884+MY7PP7ly5EGOnU6PJK2RsT7rrfylcBmZilbtTjD86+8zc9//eqovq8DwMwsZcsvns2ksU2jftN4B4CZWcrGNzfy2YX5BnFvjGKDOAeAmVkF6MxmeOe9E3x3x+g1iHMAmJlVgAVtU7hw1iTWjeJhIAeAmVkFkETn4gw7XnqdPS+PToM4B4CZWYX4bYO4UdoLcACYmVWI6ROa+YN5o9cgzgFgZlZBVmYzvHr0GI/tHvkGcQ4AM7MKcln7yQZxI38YyAFgZlZBGhvE9Yva+PGzvSPeIM4BYGZWYVZm20alQZwDwMyswpx75gSWnjed9bmXOHFi5Bp2OgDMzCrQqsUZXhjhBnEOADOzCrTsQ/kGcRtG8GSwA8DMrAKNb27k6oVns+npkWsQlygAJC2TtEdSt6RbB3hdkm4rvL5TUkepuZKmS3pU0nOF/04rz5LMzGrDyQZxD+8YmVuplwwASY3A7cByYB5wo6R5RcOWA+2Fx2rgjgRzbwUei4h24LHCczMzK5jfNoWLzprE+q6ROQyUZA9gCdAdEXsj4hhwL7CiaMwK4J7I2wJMlTS7xNwVwN8Wfv5b4HPDW4qZWW2RRGc2w46eIzzz8htl//OTBEAr0D9+egrbkowZau6siDgAUPjvzIHeXNJqSTlJud7e3gTlmpnVjs9d2splF7TwXl/5vw7alGDMQHcoLq5ksDFJ5g4pItYCayF/U/hTmWtmVu2mT2jmnj9cMiJ/dpI9gB4g0+95G1B8RmKwMUPNPVg4TEThv4eSl21mZsOVJAC6gHZJcyU1AzcAG4vGbARuKnwbaClwpHBYZ6i5G4EvFH7+AvDQMNdiZmanoOQhoIjok3QLsBloBO6OiF2S1hRevxPYBFwFdANvA18cam7hj/46sF7Sl4AXgZVlXZmZmQ1JEdVzWD2bzUYul0u7DDOzqiJpa0Rki7f7SmAzszrlADAzq1MOADOzOuUAMDOrU1V1ElhSL/DCaU6fARwuYznVwGuuD15zfRjOms+NiJbijVUVAMMhKTfQWfBa5jXXB6+5PozEmn0IyMysTjkAzMzqVD0FwNq0C0iB11wfvOb6UPY11805ADMz+131tAdgZmb9OADMzOpUzQXAcG5gX60SrPnzhbXulPRTSQvSqLOcSq2537jFko5Lun406yu3JOuVdLmk7ZJ2SfrxaNdYbgn+XU+R9LCkHYU1fzGNOstJ0t2SDkl6epDXy/v5FRE18yDfcvpXwHlAM7ADmFc05irge+TvVrYU+HnadY/Cmn8PmFb4eXk9rLnfuB+Sb1d+fdp1j/Df8VTgl8A5hecz0657FNb874D/XPi5BXgVaE679mGu+zKgA3h6kNfL+vlVa3sAw7mBfbUqueaI+GlEvFZ4uoX8ndmqWZK/Z4A/Ae6n+u82l2S9/xR4ICJeBIiIelhzAJMkCZhIPgD6RrfM8oqIJ8ivYzBl/fyqtQAYzg3sq9WprudL5H+DqGYl1yypFbgGuHMU6xopSf6OLwCmSfqRpK2Sbhq16kZGkjX/NfBB8reZfQr4s4g4MTrlpaasn19JbgpfTYZzA/tqlXg9kq4gHwAfHdGKRl6SNf9X4CsRcTz/C2JVS7LeJmAR8AlgPPAzSVsi4tmRLm6EJFnzPwG2A1cCHwAelfSTiHhjhGtLU1k/v2otAIZzA/tqlWg9kuYDfwMsj4hXRqm2kZJkzVng3sKH/wzgKkl9EfHgqFRYXkn/XR+OiKPAUUlPAAuAag2AJGv+IvD1yB8c75b0a+Ai4BejU2Iqyvr5VWuHgIZzA/tqVXLNks4BHgD+WRX/RthfyTVHxNyImBMRc4D7gJur9MMfkv27fgj4mKQmSWcAHwZ2j3Kd5ZRkzS+S3+NB0izgQmDvqFY5+sr6+VVTewAxjBvYV6uEa/4acCbw3wu/EfdFFXdSTLjmmpFkvRGxW9L3gZ3ACeBvImLArxJWg4R/x38J/C9JT5E/NPKViKjqFtGSvg1cDsyQ1AP8e2AMjMznl1tBmJnVqVo7BGRmZgk5AMzM6pQDwMysTjkAzMzqlAPAzKxOOQDMzOqUA8DMrE79f+P6qrEmbe8oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.linspace(0.,1,5)\n",
    "f = combined_cos(0.75, LR, LR, LR/100000)\n",
    "plt.plot(p, [f(o) for o in p]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_cos = combined_cos(0.75, LR, LR, LR/100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1: * TrainLoss 1.597  * TrainAcc@1 54.557  * ValLoss 1.480              * ValAcc@1 60.713  * Time 68.00\n",
      "2: * TrainLoss 1.216  * TrainAcc@1 71.254  * ValLoss 1.275              * ValAcc@1 70.217  * Time 64.98\n",
      "3: * TrainLoss 1.087  * TrainAcc@1 76.946  * ValLoss 1.045              * ValAcc@1 79.720  * Time 65.06\n",
      "4: * TrainLoss 0.997  * TrainAcc@1 80.885  * ValLoss 1.002              * ValAcc@1 81.045  * Time 65.09\n",
      "5: * TrainLoss 0.849  * TrainAcc@1 87.454  * ValLoss 0.862              * ValAcc@1 86.930  * Time 65.12\n"
     ]
    }
   ],
   "source": [
    "current_batch = 1 \n",
    "\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "    # training\n",
    "    model.train()\n",
    "    \n",
    "    start = time.time()\n",
    "    losses = AverageMeter('Loss', ':.4e')\n",
    "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
    "    \n",
    "    for i, (images, target) in enumerate(train_loader):\n",
    "        images = images.cuda(non_blocking=True) # non-blocking here with a single GPU will likely do nothing\n",
    "        target = target.cuda(non_blocking=True) # non-blocking here makes sense - we can start\n",
    "                                                          # doing something else before target is needed\n",
    "\n",
    "        # compute output\n",
    "        output = model(images)\n",
    "        loss = criterion(output, target)\n",
    "        \n",
    "        # measure accuracy and record loss\n",
    "        losses.update(loss.item(), images.size(0))\n",
    "        acc1 = accuracy(output, target, topk=(1,))\n",
    "        top1.update(acc1[0].item(), images.size(0))\n",
    "\n",
    "        # compute gradient and do SGD step\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        new_lr = flat_cos(current_batch / (NUM_EPOCHS * len(train_loader)))\n",
    "        optimizer.set_hyper('lr', new_lr)\n",
    "        current_batch += 1\n",
    "    \n",
    "    # switch to evaluate mode\n",
    "    model.eval()\n",
    "\n",
    "    val_losses = AverageMeter('Loss', ':.4e')\n",
    "    val_top1 = AverageMeter('Acc@1', ':6.2f')\n",
    "    with torch.no_grad():\n",
    "        for i, (images, target) in enumerate(val_loader):\n",
    "            images = images.cuda(non_blocking=True)\n",
    "            target = target.cuda(non_blocking=True)\n",
    "\n",
    "            # compute output\n",
    "            output = model(images)\n",
    "            loss = criterion(output, target)\n",
    "\n",
    "            # measure accuracy and record loss\n",
    "            val_losses.update(loss.item(), images.size(0))\n",
    "            acc1 = accuracy(output, target, topk=(1,))\n",
    "            val_top1.update(acc1[0].item(), images.size(0))\n",
    "            \n",
    "    # measure elapsed time\n",
    "    end = time.time()\n",
    "    elapsed = end - start\n",
    "    \n",
    "    print(f'{epoch+1}: * TrainLoss {losses.avg:.3f}  * TrainAcc@1 {top1.avg:.3f}  * ValLoss {val_losses.avg:.3f}  \\\n",
    "            * ValAcc@1 {val_top1.avg:.3f}  * Time {elapsed:.2f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}