0.1. Transform_dataset_to_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "\"\"\"\n",
    "Format Dataset: (list of lists) \n",
    "    [\n",
    "        [context_0, context_1, ..., context_N, response], # sample_0\n",
    "        [context_0, context_1, ..., context_N, response], # sample_1\n",
    "        ...,\n",
    "        [context_0, context_1, ..., context_N, response] # sample_L\n",
    "    ]\n",
    "\"\"\""
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\nFormat Dataset: (list of lists) \\n    [\\n        [context_0, context_1, ..., context_N, response], # sample_0\\n        [context_0, context_1, ..., context_N, response], # sample_1\\n        ...,\\n        [context_0, context_1, ..., context_N, response] # sample_L\\n    ]\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# tokenizer\n",
    "from transformers import (\n",
    "    MODEL_WITH_LM_HEAD_MAPPING,\n",
    "    WEIGHTS_NAME,\n",
    "    AdamW,\n",
    "    AutoConfig,\n",
    "    AutoModelWithLMHead,\n",
    "    AutoTokenizer,\n",
    "    PreTrainedModel,\n",
    "    PreTrainedTokenizer,\n",
    "    get_linear_schedule_with_warmup,\n",
    ")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "PATH_DATASET = \"0.0. prepare_dataset.csv\"\n",
    "\n",
    "# Model\n",
    "DialogGPT_MODEL = \"microsoft/DialoGPT-small\" "
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "# Instance Model & Tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(DialogGPT_MODEL)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "df = pd.read_csv(PATH_DATASET)\n",
    "df.head(5)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context_0</th>\n",
       "      <th>context_1</th>\n",
       "      <th>context_2</th>\n",
       "      <th>context_3</th>\n",
       "      <th>context_4</th>\n",
       "      <th>context_5</th>\n",
       "      <th>context_6</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Morty! You gotta come on. Jus'... you gotta co...</td>\n",
       "      <td>What, Rick? What’s going on?</td>\n",
       "      <td>I got a surprise for you, Morty.</td>\n",
       "      <td>It's the middle of the night. What are you tal...</td>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What, Rick? What’s going on?</td>\n",
       "      <td>I got a surprise for you, Morty.</td>\n",
       "      <td>It's the middle of the night. What are you tal...</td>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "      <td>Yeah, Rick... I-it's great. Is this the surprise?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I got a surprise for you, Morty.</td>\n",
       "      <td>It's the middle of the night. What are you tal...</td>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "      <td>Yeah, Rick... I-it's great. Is this the surprise?</td>\n",
       "      <td>Morty. I had to... I had to do it. I had— I ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It's the middle of the night. What are you tal...</td>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "      <td>Yeah, Rick... I-it's great. Is this the surprise?</td>\n",
       "      <td>Morty. I had to... I had to do it. I had— I ha...</td>\n",
       "      <td>What?! A bomb?!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Come on, I got a surprise for you.  Come on, h...</td>\n",
       "      <td>Ow! Ow! You're tugging me too hard!</td>\n",
       "      <td>We gotta go, gotta get outta here, come on. Go...</td>\n",
       "      <td>What do you think of this... flying vehicle, M...</td>\n",
       "      <td>Yeah, Rick... I-it's great. Is this the surprise?</td>\n",
       "      <td>Morty. I had to... I had to do it. I had— I ha...</td>\n",
       "      <td>What?! A bomb?!</td>\n",
       "      <td>We're gonna drop it down there just get a whol...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           context_0  \\\n",
       "0  Morty! You gotta come on. Jus'... you gotta co...   \n",
       "1                       What, Rick? What’s going on?   \n",
       "2                   I got a surprise for you, Morty.   \n",
       "3  It's the middle of the night. What are you tal...   \n",
       "4  Come on, I got a surprise for you.  Come on, h...   \n",
       "\n",
       "                                           context_1  \\\n",
       "0                       What, Rick? What’s going on?   \n",
       "1                   I got a surprise for you, Morty.   \n",
       "2  It's the middle of the night. What are you tal...   \n",
       "3  Come on, I got a surprise for you.  Come on, h...   \n",
       "4                Ow! Ow! You're tugging me too hard!   \n",
       "\n",
       "                                           context_2  \\\n",
       "0                   I got a surprise for you, Morty.   \n",
       "1  It's the middle of the night. What are you tal...   \n",
       "2  Come on, I got a surprise for you.  Come on, h...   \n",
       "3                Ow! Ow! You're tugging me too hard!   \n",
       "4  We gotta go, gotta get outta here, come on. Go...   \n",
       "\n",
       "                                           context_3  \\\n",
       "0  It's the middle of the night. What are you tal...   \n",
       "1  Come on, I got a surprise for you.  Come on, h...   \n",
       "2                Ow! Ow! You're tugging me too hard!   \n",
       "3  We gotta go, gotta get outta here, come on. Go...   \n",
       "4  What do you think of this... flying vehicle, M...   \n",
       "\n",
       "                                           context_4  \\\n",
       "0  Come on, I got a surprise for you.  Come on, h...   \n",
       "1                Ow! Ow! You're tugging me too hard!   \n",
       "2  We gotta go, gotta get outta here, come on. Go...   \n",
       "3  What do you think of this... flying vehicle, M...   \n",
       "4  Yeah, Rick... I-it's great. Is this the surprise?   \n",
       "\n",
       "                                           context_5  \\\n",
       "0                Ow! Ow! You're tugging me too hard!   \n",
       "1  We gotta go, gotta get outta here, come on. Go...   \n",
       "2  What do you think of this... flying vehicle, M...   \n",
       "3  Yeah, Rick... I-it's great. Is this the surprise?   \n",
       "4  Morty. I had to... I had to do it. I had— I ha...   \n",
       "\n",
       "                                           context_6  \\\n",
       "0  We gotta go, gotta get outta here, come on. Go...   \n",
       "1  What do you think of this... flying vehicle, M...   \n",
       "2  Yeah, Rick... I-it's great. Is this the surprise?   \n",
       "3  Morty. I had to... I had to do it. I had— I ha...   \n",
       "4                                    What?! A bomb?!   \n",
       "\n",
       "                                            response  \n",
       "0  What do you think of this... flying vehicle, M...  \n",
       "1  Yeah, Rick... I-it's great. Is this the surprise?  \n",
       "2  Morty. I had to... I had to do it. I had— I ha...  \n",
       "3                                    What?! A bomb?!  \n",
       "4  We're gonna drop it down there just get a whol...  "
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "TEST_SIZE = 0.1"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Split Dataset in train & test"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "df_train, df_val = train_test_split(df,test_size=TEST_SIZE, random_state=0)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "# transform dataset\n",
    "def construct_conv(row, tokenizer, eos = True):\n",
    "    \"\"\" Add EOS token to each sentecenes from row --> tokenized ---> flatten tokenized row\n",
    "    Args:\n",
    "        row (list of sentences): the first N elements is the context and the last element is the response or target\n",
    "        tokenizer (object): tokenizer from respective model\n",
    "        eos (bool, optional): end of sentence. Defaults to True.\n",
    "\n",
    "    Returns:\n",
    "        (list of ints): [len(flatten(row))] (shape is variable because not have padding) flatten row tokenized \n",
    "            >>>>  [651,5513,86,24905,287,...,640,284,651,5513,86,24905] --> 2495=eos_token\n",
    "    \"\"\"\n",
    "    flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "    conv = [tokenizer.encode(x) + [tokenizer.eos_token_id] for x in row]\n",
    "    conv = flatten(conv) \n",
    "    return conv\n",
    "\n",
    "\n",
    "import os\n",
    "from torch.utils.data import Dataset\n",
    "import logging\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "class ConversationDataset(Dataset):\n",
    "    def __init__(self, tokenizer, args, df, block_size=512):\n",
    "\n",
    "        block_size = block_size - (tokenizer.max_len - tokenizer.max_len_single_sentence)\n",
    "\n",
    "        directory = \"cached\"\n",
    "        cached_features_file = os.path.join(\n",
    "            directory, args.model_type + \"_cached_lm_\" + str(block_size)\n",
    "        )\n",
    "\n",
    "        if os.path.exists(cached_features_file) and not args.overwrite_cache:\n",
    "            logger.info(\"Loading features from cached file %s\", cached_features_file)\n",
    "            with open(cached_features_file, \"rb\") as handle:\n",
    "                self.examples = pickle.load(handle)\n",
    "        else:\n",
    "            logger.info(\"Creating features from dataset file at %s\", directory)\n",
    "\n",
    "            self.examples = []\n",
    "            for _, row in df.iterrows():\n",
    "                conv = construct_conv(row, tokenizer)\n",
    "                self.examples.append(conv)\n",
    "\n",
    "            logger.info(\"Saving features into cached file %s\", cached_features_file)\n",
    "            with open(cached_features_file, \"wb\") as handle:\n",
    "                pickle.dump(self.examples, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.examples)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        return torch.tensor(self.examples[item], dtype=torch.long)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "model_type = 'gpt2'\n",
    "block_size=512\n",
    "\n",
    "block_size = block_size - (tokenizer.max_len - tokenizer.max_len_single_sentence)\n",
    "\n",
    "directory = \"cached\"\n",
    "cached_features_file = os.path.join(\n",
    "    directory, model_type + \"_cached_lm_\" + str(block_size)\n",
    "    )"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "AttributeError",
     "evalue": "'GPT2TokenizerFast' object has no attribute 'max_len'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-7b65bcefbf1b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mblock_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mblock_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mblock_size\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len_single_sentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mdirectory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"cached\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'GPT2TokenizerFast' object has no attribute 'max_len'"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "tokenizer.max_len"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "AttributeError",
     "evalue": "'GPT2TokenizerFast' object has no attribute 'max_len'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-be6172d0da6f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'GPT2TokenizerFast' object has no attribute 'max_len'"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.5",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "4a64d8e05436eb44142eb50eb52ba7fe11bf3b9fdb34fcbbedc3b49239359503"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}