vit_finetuner_solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02126f3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mvetka925\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "Global seed set to 100\n"
     ]
    }
   ],
   "source": [
    "from transformers import ViTFeatureExtractor\n",
    "from torch.utils.data import Dataset\n",
    "import pandas as pd\n",
    "import pytorch_lightning as pl\n",
    "from transformers import ViTForImageClassification, AdamW\n",
    "import torch.nn.functional as F\n",
    "import cv2\n",
    "import torch\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torchvision.transforms import (CenterCrop, \n",
    "                                    Compose, \n",
    "                                    Normalize, \n",
    "                                    RandomHorizontalFlip,\n",
    "                                    RandomResizedCrop, \n",
    "                                    Resize, \n",
    "                                    ToTensor)\n",
    "import wandb\n",
    "from pytorch_lightning.loggers import WandbLogger\n",
    "import torchmetrics\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "import numpy as np\n",
    "from torchvision import models\n",
    "\n",
    "wandb.login()\n",
    "pl.seed_everything(100)\n",
    "\n",
    "TRAIN_IMGS_PATH = './train'\n",
    "TEST_IMGS_PATH = './test'\n",
    "TRAIN_DF_PATH = './train.csv'\n",
    "\n",
    "MODEL_VESRION = 'google/vit-base-patch16-224-in21k'\n",
    "IAMGE_SIZE = 224"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47a84e84",
   "metadata": {},
   "source": [
    "# Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "944e99e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "image = cv2.imread(f\"{TRAIN_IMGS_PATH}/220301070305_0e13309ae71ffc37ba629d19f46e0784.jpg\")\n",
    "image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "image0 = cv2.imread(f\"{TRAIN_IMGS_PATH}/220301070430_df1a737a5abe3707b424fbe9d1d92300.jpg\")\n",
    "image0 = cv2.cvtColor(image0, cv2.COLOR_BGR2RGB)\n",
    "image1 = cv2.imread(f\"{TRAIN_IMGS_PATH}/220301070439_91576ea7fbb6567ff743e271662f9d06.jpg\")\n",
    "image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "def visualize(image):\n",
    "    plt.figure(figsize=(10, 10))\n",
    "    plt.axis('off')\n",
    "    plt.imshow(image)\n",
    "\n",
    "\n",
    "feature_extractor = ViTFeatureExtractor(MODEL_VESRION)\n",
    "normalize = A.Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)\n",
    "\n",
    "\n",
    "_train_transforms = A.Compose([\n",
    "    A.Resize(224, 224),\n",
    "    normalize,\n",
    "     ToTensorV2(),\n",
    "])\n",
    "\n",
    "\n",
    "_val_transforms = A.Compose(\n",
    "        [\n",
    "            A.Resize(224, 224),\n",
    "            normalize,\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "\n",
    "def init_transforms(image_size):\n",
    "    normalize = A.Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)\n",
    "    albumentations_transform_oneof = A.Compose([\n",
    "        A.Resize(image_size, image_size),\n",
    "\n",
    "        normalize,\n",
    "         ToTensorV2(),\n",
    "    ])\n",
    "    \n",
    "    _val_transforms = A.Compose(\n",
    "        [\n",
    "             A.Resize(image_size, image_size),\n",
    "            normalize,\n",
    "            ToTensorV2(),\n",
    "        ]\n",
    "    )\n",
    "    \n",
    "    return albumentations_transform_oneof, _val_transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99f1bda4",
   "metadata": {},
   "source": [
    "# Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b304398",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class ImageDataset(Dataset):\n",
    "    def __init__(self, data_df, transform=None, add_transform=None):\n",
    "\n",
    "        self.data_df = data_df\n",
    "        self.transform = transform\n",
    "        self.add_transform = add_transform\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        image_name, label = self.data_df.iloc[idx]['ID_img'], self.data_df.iloc[idx]['class']\n",
    "    \n",
    "        image = cv2.imread(f\"{TRAIN_IMGS_PATH}/{image_name}\")\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "        if self.transform:\n",
    "            image = self.transform(image=image)['image']\n",
    "        \n",
    "        return image, torch.tensor(label).long()\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data_df)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0457e81",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "data_df = pd.read_csv(TRAIN_DF_PATH)\n",
    "data_df['additional'] = 0\n",
    "\n",
    "data_df = data_df.sample(frac=1, random_state=44).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "48da96a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1.0: 292, 2.0: 149, 0.0: 102}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_counts = data_df['class'].value_counts().to_dict()\n",
    "class_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfc28c2a",
   "metadata": {},
   "source": [
    "# Custom ROC AUC metric for PL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b3884aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchmetrics import Metric\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from typing import Optional, Any\n",
    "import torch\n",
    "\n",
    "def one_label_to_many(preds):\n",
    "    result = []\n",
    "    for p in preds:\n",
    "        many = [0, 0, 0]\n",
    "        many[p] = 1\n",
    "        result.append(many)\n",
    "    return result\n",
    "\n",
    "class ROCAUC(Metric):\n",
    "    def __init__(\n",
    "        self,\n",
    "        average='macro',\n",
    "        compute_on_step: bool = True,\n",
    "        dist_sync_on_step: bool = False,\n",
    "        process_group: Optional[Any] = None,\n",
    "    ):\n",
    "        super().__init__(\n",
    "            compute_on_step=compute_on_step,\n",
    "            dist_sync_on_step=dist_sync_on_step,\n",
    "            process_group=process_group,\n",
    "        )\n",
    "\n",
    "        self.average = average\n",
    "        self.add_state(\"preds\", default=[], dist_reduce_fx=None)\n",
    "        self.add_state(\"target\", default=[], dist_reduce_fx=None)\n",
    "\n",
    "    def update(self, preds: torch.Tensor, target: torch.Tensor):\n",
    "        \"\"\"\n",
    "        Update state with predictions and targets.\n",
    "\n",
    "        Args:\n",
    "            preds: Predictions from model\n",
    "            target: Ground truth values\n",
    "        \"\"\"\n",
    "        self.preds.append(preds)\n",
    "        self.target.append(target)\n",
    "\n",
    "    def compute(self):\n",
    "        \n",
    "        preds = np.array(one_label_to_many(torch.cat(self.preds, dim=0).cpu().numpy()))\n",
    "        target = torch.cat(self.target, dim=0).cpu().numpy().astype(int)\n",
    "        score = roc_auc_score(target, preds, average=self.average, multi_class='ovo', labels=[0,1,2])\n",
    "        return torch.tensor([score], dtype=torch.float32, device='cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8765fc3c",
   "metadata": {},
   "source": [
    "# VIT module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "15cc7b92",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VITFineTuner(pl.LightningModule):\n",
    "    def __init__(self, model_version, num_labels, train_dataset, val_dataset, batch_size):\n",
    "        super(VITFineTuner, self).__init__()\n",
    "        \n",
    "        self.vit = ViTForImageClassification.from_pretrained(model_version,\n",
    "                                                              num_labels=num_labels)\n",
    "        \n",
    "\n",
    "        self.train_dataset = train_dataset\n",
    "        self.val_dataset = val_dataset\n",
    "\n",
    "        self.batch_size = batch_size\n",
    "        self.num_labels = num_labels\n",
    "\n",
    "        self.save_hyperparameters()\n",
    "        \n",
    "\n",
    "        self.train_rocauc = ROCAUC(average='macro')\n",
    "        self.val_rocauc = ROCAUC( average='macro')\n",
    "        \n",
    "    def forward(self, X):\n",
    "        outputs = self.vit(X)\n",
    "        return outputs.logits\n",
    "    \n",
    "    def common_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        logits = self(x)\n",
    "        loss = F.cross_entropy(logits, y)\n",
    "        \n",
    "        if self.num_labels >= 1:\n",
    "            preds = torch.argmax(logits, axis=1)\n",
    "        elif self.num_labels == 1:\n",
    "            preds = logits.squeeze()\n",
    "        \n",
    "        return preds, y, loss, logits\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        preds, y, loss, _ = self.common_step(batch, batch_idx)\n",
    "\n",
    "        self.log('train/loss', loss, on_epoch=True)\n",
    "        self.train_rocauc(y, preds)\n",
    "        self.log('train/rocauc', self.train_rocauc, on_epoch=True)\n",
    "        return loss\n",
    "    \n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        preds, y, loss, logits = self.common_step(batch, batch_idx)\n",
    "\n",
    "        self.val_rocauc(y, preds)\n",
    "        self.log(\"validation/loss_epoch\", loss)\n",
    "        self.log(\"validation/roc_auc_epoch\", self.val_rocauc)\n",
    "        return preds, y, logits\n",
    "    \n",
    "    def validation_epoch_end(self, validation_step_outputs):\n",
    "        preds = torch.cat([e[0] for e in validation_step_outputs]).cpu().detach().numpy()\n",
    "        y = torch.cat([e[1] for e in validation_step_outputs]).cpu().detach().numpy()\n",
    "\n",
    "        logits = torch.cat([e[2] for e in validation_step_outputs]).cpu().detach().numpy()\n",
    "        self.logger.experiment.log({\"conf_mat\" : wandb.plot.confusion_matrix( probs=None, y_true=y, preds=preds, class_names=[\"0\", \"1\", \"2\"])})\n",
    "        self.logger.experiment.log(\n",
    "            {\"validation/logits\": wandb.Histogram(logits),\n",
    "             \"global_step\": self.global_step}, commit=False\n",
    "        )\n",
    "\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        return AdamW(self.parameters(), lr=5e-5)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        train_loader = torch.utils.data.DataLoader(self.train_dataset, \n",
    "                                                   batch_size=self.batch_size, \n",
    "                                                  shuffle=True, pin_memory=True)\n",
    "        return train_loader\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        val_loader = torch.utils.data.DataLoader(self.val_dataset, batch_size=self.batch_size)\n",
    "        return val_loader\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea9bb083",
   "metadata": {},
   "source": [
    "# Draw images in wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9e88db2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImageLogger(pl.Callback):\n",
    "    def __init__(self, val_samples, num_samples=None):\n",
    "        super().__init__()\n",
    "        if not num_samples:\n",
    "            num_samples = len(val_samples[0])\n",
    "        self.val_imgs, self.val_labels = val_samples\n",
    "        self.val_imgs = self.val_imgs[:num_samples]\n",
    "        self.val_labels = self.val_labels[:num_samples]\n",
    "\n",
    "    def on_validation_epoch_end(self, trainer, pl_module):\n",
    "        val_imgs = self.val_imgs.to(device=pl_module.device)\n",
    "        logits = pl_module(val_imgs)\n",
    "        preds = torch.argmax(logits, 1)\n",
    "        \n",
    "        mistakes = [preds.detach().cpu() != self.val_labels.detach().cpu()]\n",
    "        trainer.logger.experiment.log({\n",
    "            \"Mistakes\": [wandb.Image(x, caption=f\"Pred: {pred}, Label: {y}\") for x, pred, y in zip(val_imgs[mistakes], \n",
    "                                                                                                   preds[mistakes], \n",
    "                                                                                                   self.val_labels[mistakes])][:10],\n",
    "            \"global_step\": trainer.global_step\n",
    "        }, commit=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50b88749",
   "metadata": {},
   "source": [
    "# Train single model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67c566bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_130926-t9z9xvn4</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_single/runs/t9z9xvn4\" target=\"_blank\">dauntless-deluge-3</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_single\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (8) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbbce4b7588d482081789dd4f82907bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='3.155 MB of 3.155 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅██</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁▇█</td></tr><tr><td>train/rocauc_step</td><td>▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅▅██</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁▅█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>24</td></tr><tr><td>train/loss_epoch</td><td>0.54029</td></tr><tr><td>train/loss_step</td><td>0.77921</td></tr><tr><td>train/rocauc_epoch</td><td>0.97897</td></tr><tr><td>train/rocauc_step</td><td>0.93182</td></tr><tr><td>trainer/global_step</td><td>23</td></tr><tr><td>validation/loss_epoch</td><td>0.57377</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.90188</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">dauntless-deluge-3</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_single/runs/t9z9xvn4\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_single/runs/t9z9xvn4</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_130926-t9z9xvn4\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import re \n",
    "\n",
    "# PARAMS\n",
    "model_version = MODEL_VESRION\n",
    "num_labels = 3\n",
    "batch_size = 32\n",
    "num_epochs = 3\n",
    "\n",
    "\n",
    "# SPLIT, BALANCE DATA\n",
    "train_df, valid_df = train_test_split(data_df, test_size=0.2, random_state=10)\n",
    "\n",
    "\n",
    "# BALANCE BY MIN\n",
    "g = train_df.groupby('class')\n",
    "train_df = g.apply(lambda x: x.sample(g.size().min(), random_state=44)).reset_index(drop=True)\n",
    "\n",
    "\n",
    "#    CREATE TORCH DATASETS\n",
    "train_dataset = ImageDataset(train_df, _train_transforms)\n",
    "valid_dataset = ImageDataset(valid_df, _val_transforms)\n",
    "samples_loader =  torch.utils.data.DataLoader(valid_dataset, batch_size=len(valid_dataset))\n",
    "\n",
    "#    INIT LOGGER\n",
    "_model_version = re.sub(r'[/\\\\]', '_', MODEL_VESRION)\n",
    "wandb_logger = WandbLogger(project=f\"garbage_cross_val_{_model_version}_run_single\")\n",
    "\n",
    "#    INIT MODEL\n",
    "finetuner = VITFineTuner(model_version, num_labels, train_dataset, valid_dataset, batch_size)\n",
    "\n",
    "#    TRAINER\n",
    "trainer = pl.Trainer(\n",
    "    logger=wandb_logger,    # W&B integration\n",
    "    log_every_n_steps=13,   # set the logging frequency\n",
    "    gpus=-1,                # use all GPUs\n",
    "    max_epochs=num_epochs,           # number of epochs\n",
    "    deterministic=True,     # keep it deterministic\n",
    "    callbacks=[ImageLogger(next(iter(samples_loader)))] # see Callbacks section\n",
    "    )\n",
    "trainer.fit(finetuner)\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bf95f01",
   "metadata": {},
   "source": [
    "# Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "acac6de2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "def model_cross_validation(data, num_folds, model_module, model_version, num_labels, batch_size, num_epochs):\n",
    "    result_metrics = []\n",
    "    \n",
    "    train_transforms, val_transforms = init_transforms(224)\n",
    "    \n",
    "    skf = KFold(num_folds, shuffle=True, random_state=44)\n",
    "    for train_index, val_index in skf.split(data['ID_img'], data['class']):\n",
    "        \n",
    "        train_df = data.iloc[train_index]\n",
    "        \n",
    "#         # BALANCE TRAIN!\n",
    "#         class_counts = train_df[train_df['class'] != 1]['class'].value_counts().to_dict()\n",
    "#         max_elements = max(class_counts.values())\n",
    "        \n",
    "#         train_df = pd.concat([train_df[train_df['class'] == 1].sample(max_elements),\n",
    "#                                train_df[train_df['class'] == 0],\n",
    "#                                train_df[train_df['class'] == 2]]).sample(frac=1).reset_index(drop=True)\n",
    "#         #######\n",
    "        \n",
    "        g = train_df.groupby('class')\n",
    "        train_df = g.apply(lambda x: x.sample(g.size().min(), random_state=44)).reset_index(drop=True)\n",
    "\n",
    "        val_df = data.iloc[val_index]\n",
    "        train_dataset = ImageDataset(train_df, train_transforms)\n",
    "        valid_dataset = ImageDataset(val_df, val_transforms)\n",
    "        samples_loader =  torch.utils.data.DataLoader(valid_dataset, batch_size=len(valid_dataset))\n",
    "        \n",
    "        _model_version = re.sub(r'[/\\\\]', '_', model_version)\n",
    "        wandb_logger = WandbLogger(project=f\"garbage_cross_val_{_model_version}_run_cv_2\")\n",
    "\n",
    "\n",
    "        vit_finetuner = model_module(model_version, num_labels, train_dataset, valid_dataset, batch_size)\n",
    "\n",
    "        trainer = pl.Trainer(\n",
    "            logger=wandb_logger,    # W&B integration\n",
    "            log_every_n_steps=13,   # set the logging frequency\n",
    "            gpus=-1,                # use all GPUs\n",
    "            max_epochs=num_epochs,           # number of epochs\n",
    "            deterministic=True,     # keep it deterministic\n",
    "            callbacks=[ImageLogger(next(iter(samples_loader)))] # see Callbacks section\n",
    "            )\n",
    "        trainer.fit(vit_finetuner)\n",
    "        wandb.finish()\n",
    "        result_metrics.append(trainer.logged_metrics)\n",
    "        \n",
    "    return result_metrics\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "03a31e92",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131029-2ngjl8ce</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2ngjl8ce\" target=\"_blank\">grateful-lake-1</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (9) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be29ee88231047bd859cfe1d5f3e3f89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='3.116 MB of 3.116 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅███</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>█▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁██</td></tr><tr><td>train/rocauc_step</td><td>▁█</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅████</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁▂█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>27</td></tr><tr><td>train/loss_epoch</td><td>0.46713</td></tr><tr><td>train/loss_step</td><td>0.36702</td></tr><tr><td>train/rocauc_epoch</td><td>0.97602</td></tr><tr><td>train/rocauc_step</td><td>1.0</td></tr><tr><td>trainer/global_step</td><td>26</td></tr><tr><td>validation/loss_epoch</td><td>0.54526</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.81511</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">grateful-lake-1</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2ngjl8ce\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2ngjl8ce</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_131029-2ngjl8ce\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131133-2f02hxx4</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2f02hxx4\" target=\"_blank\">whole-dream-2</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (8) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "201364ad5f9842ca8f5edacb8655b5ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='3.146 MB of 3.146 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅██</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁▇█</td></tr><tr><td>train/rocauc_step</td><td>▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅▅██</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁██</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>24</td></tr><tr><td>train/loss_epoch</td><td>0.56024</td></tr><tr><td>train/loss_step</td><td>0.77344</td></tr><tr><td>train/rocauc_epoch</td><td>0.97101</td></tr><tr><td>train/rocauc_step</td><td>0.92917</td></tr><tr><td>trainer/global_step</td><td>23</td></tr><tr><td>validation/loss_epoch</td><td>0.59578</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.8919</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">whole-dream-2</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2f02hxx4\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/2f02hxx4</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_131133-2f02hxx4\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131232-3m9o01sy</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/3m9o01sy\" target=\"_blank\">blooming-pine-3</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (8) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "542e0bcd173c435781057c814c77fc77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='2.934 MB of 2.934 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅██</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁▇█</td></tr><tr><td>train/rocauc_step</td><td>▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅▅██</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁▄█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>24</td></tr><tr><td>train/loss_epoch</td><td>0.53077</td></tr><tr><td>train/loss_step</td><td>0.75505</td></tr><tr><td>train/rocauc_epoch</td><td>0.96311</td></tr><tr><td>train/rocauc_step</td><td>0.925</td></tr><tr><td>trainer/global_step</td><td>23</td></tr><tr><td>validation/loss_epoch</td><td>0.52923</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.89896</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">blooming-pine-3</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/3m9o01sy\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/3m9o01sy</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_131232-3m9o01sy\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131333-ib3pklen</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/ib3pklen\" target=\"_blank\">pleasant-violet-4</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (8) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba215f71980743ceb6f590b2c0669b45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='3.252 MB of 3.252 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅██</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁▇█</td></tr><tr><td>train/rocauc_step</td><td>▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅▅██</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁▃█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>24</td></tr><tr><td>train/loss_epoch</td><td>0.53567</td></tr><tr><td>train/loss_step</td><td>0.71678</td></tr><tr><td>train/rocauc_epoch</td><td>0.95816</td></tr><tr><td>train/rocauc_step</td><td>0.93304</td></tr><tr><td>trainer/global_step</td><td>23</td></tr><tr><td>validation/loss_epoch</td><td>0.59951</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.8969</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">pleasant-violet-4</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/ib3pklen\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/ib3pklen</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_131333-ib3pklen\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131435-js5e31ep</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/js5e31ep\" target=\"_blank\">noble-bush-5</a></strong> to <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (8) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00d521ec42c34785adfc6d338b8aa0f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='3.142 MB of 3.142 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▅▅▅██</td></tr><tr><td>global_step</td><td>▁▃▆█</td></tr><tr><td>train/loss_epoch</td><td>█▄▁</td></tr><tr><td>train/loss_step</td><td>▁</td></tr><tr><td>train/rocauc_epoch</td><td>▁██</td></tr><tr><td>train/rocauc_step</td><td>▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▃▃▅▅▅██</td></tr><tr><td>validation/loss_epoch</td><td>█▄▁</td></tr><tr><td>validation/roc_auc_epoch</td><td>▁▆█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>2</td></tr><tr><td>global_step</td><td>24</td></tr><tr><td>train/loss_epoch</td><td>0.58655</td></tr><tr><td>train/loss_step</td><td>0.80847</td></tr><tr><td>train/rocauc_epoch</td><td>0.96601</td></tr><tr><td>train/rocauc_step</td><td>0.91667</td></tr><tr><td>trainer/global_step</td><td>23</td></tr><tr><td>validation/loss_epoch</td><td>0.59172</td></tr><tr><td>validation/roc_auc_epoch</td><td>0.91591</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">noble-bush-5</strong>: <a href=\"https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/js5e31ep\" target=\"_blank\">https://wandb.ai/vetka925/garbage_cross_val_google_vit-base-patch16-224-in21k_run_cv_2/runs/js5e31ep</a><br/>Synced 5 W&B file(s), 44 media file(s), 4 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20220702_131435-js5e31ep\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cv_scores = model_cross_validation(data_df, 5, VITFineTuner, MODEL_VESRION, 3, 32, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc9b2599",
   "metadata": {},
   "source": [
    "# CV Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5bf49313",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average ROC AUC:  0.88375443\n"
     ]
    }
   ],
   "source": [
    "print('Average ROC AUC: ', np.mean([ e['validation/roc_auc_epoch'].detach().cpu().numpy() for e in cv_scores]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e19a219c",
   "metadata": {},
   "source": [
    "# Train full data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e904d4b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google/vit-base-patch16-224-in21k were not used when initializing ViTForImageClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing ViTForImageClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ViTForImageClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\torchmetrics\\utilities\\prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has\n",
      "                not been set for this class (ROCAUC). The property determines if `update` by\n",
      "                default needs access to the full metric state. If this is not the case, significant speedups can be\n",
      "                achieved and we recommend setting this to `False`.\n",
      "                We provide an checking function\n",
      "                `from torchmetrics.utilities import check_forward_no_full_state`\n",
      "                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,\n",
      "                default for now) or if `full_state_update=False` can be used safely.\n",
      "                \n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href=\"https://wandb.me/wandb-init\" target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.20 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.18"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>D:\\Nextcloud\\Projects\\garbage\\wandb\\run-20220702_131537-31d5puuv</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/vetka925/train_run/runs/31d5puuv\" target=\"_blank\">graceful-dream-2</a></strong> to <a href=\"https://wandb.ai/vetka925/train_run\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "\n",
      "  | Name         | Type                      | Params\n",
      "-----------------------------------------------------------\n",
      "0 | vit          | ViTForImageClassification | 85.8 M\n",
      "1 | train_rocauc | ROCAUC                    | 0     \n",
      "2 | val_rocauc   | ROCAUC                    | 0     \n",
      "-----------------------------------------------------------\n",
      "85.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "85.8 M    Total params\n",
      "343.204   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:240: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\pytorch_lightning\\trainer\\trainer.py:1933: PossibleUserWarning: The number of training batches (10) is smaller than the logging interval Trainer(log_every_n_steps=13). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a59d515926d24543996efce647b5699c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py:495: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "d:\\nextcloud\\projects\\garbage\\venv\\lib\\site-packages\\numpy\\core\\_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "data_df = pd.read_csv(TRAIN_DF_PATH)\n",
    "data_df['additional'] = 0\n",
    "\n",
    "\n",
    "# BALANCE BY MIN\n",
    "g = data_df.groupby('class')\n",
    "data_df = g.apply(lambda x: x.sample(g.size().min(), random_state=44)).reset_index(drop=True)\n",
    "\n",
    "train_dataset = ImageDataset(data_df, _train_transforms)\n",
    "val_dataset = ImageDataset(data_df.iloc[:33], _val_transforms)\n",
    "\n",
    "samples_loader =  torch.utils.data.DataLoader(val_dataset, batch_size=len(val_dataset))\n",
    "\n",
    "finetuner = VITFineTuner(MODEL_VESRION, 3, train_dataset, val_dataset, 32)\n",
    "\n",
    "\n",
    "wandb_logger = WandbLogger(project=f\"train_run\")\n",
    "trainer = pl.Trainer(\n",
    "            logger=wandb_logger,    # W&B integration\n",
    "            log_every_n_steps=13,   # set the logging frequency\n",
    "            gpus=-1,                # use all GPUs\n",
    "            max_epochs=3,           # number of epochs\n",
    "            deterministic=True,     # keep it deterministic\n",
    "            callbacks=[ImageLogger(next(iter(samples_loader)))] # see Callbacks section\n",
    "            )\n",
    "trainer.fit(finetuner)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22703406",
   "metadata": {},
   "source": [
    "# Submit "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ccb77126",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"./sample_solution.csv\")\n",
    "test_df = test_df.drop([\"class\"], axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aa5da836",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestImageDataset(Dataset):\n",
    "    def __init__(self, data_df, transform=None):\n",
    "        self.data_df = data_df\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        image_name = self.data_df.iloc[idx]['ID_img']\n",
    "        \n",
    "\n",
    "        image = cv2.imread(f\"{TEST_IMGS_PATH}/{image_name}.jpg\")\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "        # МОЖНО ЗАПРИНТИТЬ КАРТИНКУ\n",
    "#         visualize(image)\n",
    "        \n",
    "        if self.transform:\n",
    "            image = self.transform(image=image)['image']\n",
    "        \n",
    "        return image\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0f0c3e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = TestImageDataset(test_df, _val_transforms)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                           batch_size=1,\n",
    "                                           shuffle=False,\n",
    "                                           pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6d0c32d3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 225/225 [01:20<00:00,  2.78it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "finetuner.eval()\n",
    "predicts = []\n",
    "\n",
    "for imgs in tqdm(test_loader):\n",
    "    \n",
    "    imgs = imgs\n",
    "    pred = finetuner(imgs)\n",
    "\n",
    "    for class_obj in pred:\n",
    "      index, max_value = max(enumerate(class_obj), key=lambda i_v: i_v[1])\n",
    "      predicts.append(index)\n",
    "        # МОЖНО ЗАПРИНТИТЬ КАРТИНКУ\n",
    "#       print(index)\n",
    "#       plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5821f696",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df['class'] = predicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "88ac27ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv('./submit.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01078f5d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}