Kamonwan/final_project
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "id": "b5cf5c32",6 "metadata": {7 "papermill": {8 "duration": 0.01025,9 "end_time": "2023-02-16T14:31:06.482504",10 "exception": false,11 "start_time": "2023-02-16T14:31:06.472254",12 "status": "completed"13 },14 "tags": []15 },16 "source": [17 "<br>\n",18 "<h1 style = \"font-size:60px; font-family:Garamond ; font-weight : normal; background-color: #f6f5f5 ; color : #fe346e; text-align: center; border-radius: 100px 100px;\">BLIP Image Captioning Training</h1>\n",19 "<br>\n",20 "\n",21 ""22 ]23 },24 {25 "cell_type": "markdown",26 "id": "bf5e3eb0",27 "metadata": {28 "papermill": {29 "duration": 0.006668,30 "end_time": "2023-02-16T14:31:06.496010",31 "exception": false,32 "start_time": "2023-02-16T14:31:06.489342",33 "status": "completed"34 },35 "tags": []36 },37 "source": [38 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\">In this notebook we will train BLIP model by Salesforce for the Image Captioning task on the DiffusionDB dataset</span> <br>\n",39 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">Reference: https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb</span>"40 ]41 },42 {43 "cell_type": "markdown",44 "id": "cdba2026",45 "metadata": {46 "papermill": {47 "duration": 0.006494,48 "end_time": "2023-02-16T14:31:06.509075",49 "exception": false,50 "start_time": "2023-02-16T14:31:06.502581",51 "status": "completed"52 },53 "tags": []54 },55 "source": [56 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Install Required Libraries</h1></span>"57 ]58 },59 {60 "cell_type": "code",61 "execution_count": 1,62 "id": "40bf4ea5",63 "metadata": {64 "_kg_hide-output": true,65 "execution": {66 "iopub.execute_input": "2023-02-16T14:31:06.524647Z",67 "iopub.status.busy": "2023-02-16T14:31:06.523876Z",68 "iopub.status.idle": "2023-02-16T14:31:31.643878Z",69 "shell.execute_reply": "2023-02-16T14:31:31.642709Z"70 },71 "papermill": {72 "duration": 25.130763,73 "end_time": "2023-02-16T14:31:31.646468",74 "exception": false,75 "start_time": "2023-02-16T14:31:06.515705",76 "status": "completed"77 },78 "tags": []79 },80 "outputs": [81 {82 "name": "stdout",83 "output_type": "stream",84 "text": [85 "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\r\n",86 "Requirement already satisfied: wandb in /opt/conda/lib/python3.7/site-packages (0.12.21)\r\n",87 "Collecting wandb\r\n",88 " Downloading wandb-0.13.10-py3-none-any.whl (2.0 MB)\r\n",89 "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",90 "\u001b[?25hRequirement already satisfied: sentry-sdk>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (1.15.0)\r\n",91 "Requirement already satisfied: GitPython>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (3.1.27)\r\n",92 "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from wandb) (59.8.0)\r\n",93 "Requirement already satisfied: requests<3,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (2.28.1)\r\n",94 "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.12.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (3.20.3)\r\n",95 "Requirement already satisfied: PyYAML in /opt/conda/lib/python3.7/site-packages (from wandb) (6.0)\r\n",96 "Requirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (5.9.2)\r\n",97 "Requirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (0.4.0)\r\n",98 "Requirement already satisfied: setproctitle in /opt/conda/lib/python3.7/site-packages (from wandb) (1.3.2)\r\n",99 "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from wandb) (4.1.1)\r\n",100 "Requirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.7/site-packages (from wandb) (1.4.4)\r\n",101 "Requirement already satisfied: Click!=8.0.0,>=7.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (8.1.3)\r\n",102 "Requirement already satisfied: pathtools in /opt/conda/lib/python3.7/site-packages (from wandb) (0.1.2)\r\n",103 "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from Click!=8.0.0,>=7.0->wandb) (6.0.0)\r\n",104 "Requirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\r\n",105 "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from GitPython>=1.0.0->wandb) (4.0.9)\r\n",106 "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (2022.12.7)\r\n",107 "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (2.1.1)\r\n",108 "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (1.26.11)\r\n",109 "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (3.3)\r\n",110 "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (3.0.5)\r\n",111 "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->Click!=8.0.0,>=7.0->wandb) (3.8.1)\r\n",112 "Installing collected packages: wandb\r\n",113 " Attempting uninstall: wandb\r\n",114 " Found existing installation: wandb 0.12.21\r\n",115 " Uninstalling wandb-0.12.21:\r\n",116 " Successfully uninstalled wandb-0.12.21\r\n",117 "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",118 "allennlp 2.10.1 requires wandb<0.13.0,>=0.10.0, but you have wandb 0.13.10 which is incompatible.\u001b[0m\u001b[31m\r\n",119 "\u001b[0mSuccessfully installed wandb-0.13.10\r\n",120 "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",121 "\u001b[0m/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\r\n",122 "Processing /kaggle/input/lavis-pretrained/salesforce-lavis/transformers-4.26.1-py3-none-any.whl\r\n",123 "Installing collected packages: transformers\r\n",124 " Attempting uninstall: transformers\r\n",125 " Found existing installation: transformers 4.20.1\r\n",126 " Uninstalling transformers-4.20.1:\r\n",127 " Successfully uninstalled transformers-4.20.1\r\n",128 "Successfully installed transformers-4.26.1\r\n",129 "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",130 "\u001b[0m/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\r\n",131 "Processing /kaggle/input/lavis-pretrained/salesforce-lavis/huggingface_hub-0.12.0-py3-none-any.whl\r\n",132 "Installing collected packages: huggingface-hub\r\n",133 " Attempting uninstall: huggingface-hub\r\n",134 " Found existing installation: huggingface-hub 0.10.1\r\n",135 " Uninstalling huggingface-hub-0.10.1:\r\n",136 " Successfully uninstalled huggingface-hub-0.10.1\r\n",137 "Successfully installed huggingface-hub-0.12.0\r\n",138 "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",139 "\u001b[0m"140 ]141 }142 ],143 "source": [144 "!pip install --upgrade wandb\n",145 "!pip install --no-index --no-deps /kaggle/input/lavis-pretrained/salesforce-lavis/transformers* \n",146 "!pip install --no-index --no-deps /kaggle/input/lavis-pretrained/salesforce-lavis/hugging*"147 ]148 },149 {150 "cell_type": "markdown",151 "id": "f290f229",152 "metadata": {153 "papermill": {154 "duration": 0.008123,155 "end_time": "2023-02-16T14:31:31.663453",156 "exception": false,157 "start_time": "2023-02-16T14:31:31.655330",158 "status": "completed"159 },160 "tags": []161 },162 "source": [163 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Import Required Libraries 📚</h1></span>"164 ]165 },166 {167 "cell_type": "code",168 "execution_count": 2,169 "id": "cb621077",170 "metadata": {171 "execution": {172 "iopub.execute_input": "2023-02-16T14:31:31.681788Z",173 "iopub.status.busy": "2023-02-16T14:31:31.681416Z",174 "iopub.status.idle": "2023-02-16T14:31:41.222887Z",175 "shell.execute_reply": "2023-02-16T14:31:41.221543Z"176 },177 "papermill": {178 "duration": 9.554096,179 "end_time": "2023-02-16T14:31:41.225822",180 "exception": false,181 "start_time": "2023-02-16T14:31:31.671726",182 "status": "completed"183 },184 "tags": []185 },186 "outputs": [],187 "source": [188 "import os\n",189 "import gc\n",190 "import copy\n",191 "import time\n",192 "import random\n",193 "import joblib\n",194 "\n",195 "# For data manipulation\n",196 "import numpy as np\n",197 "import pandas as pd\n",198 "\n",199 "# Pytorch Imports\n",200 "import torch\n",201 "import torch.nn as nn\n",202 "import torch.optim as optim\n",203 "from torch.optim import lr_scheduler\n",204 "from torch.utils.data import Dataset, DataLoader\n",205 "\n",206 "# Utils\n",207 "from tqdm import tqdm\n",208 "from collections import defaultdict\n",209 "\n",210 "# For Transformer Models\n",211 "from transformers import AutoProcessor, AdamW\n",212 "from transformers import BlipForConditionalGeneration\n",213 "\n",214 "# For colored terminal text\n",215 "from colorama import Fore, Back, Style\n",216 "b_ = Fore.BLUE\n",217 "y_ = Fore.YELLOW\n",218 "sr_ = Style.RESET_ALL\n",219 "\n",220 "# Suppress warnings\n",221 "import warnings\n",222 "warnings.filterwarnings(\"ignore\")\n",223 "\n",224 "# For descriptive error messages\n",225 "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n",226 "os.environ['TOKENIZERS_PARALLELISM'] = \"False\""227 ]228 },229 {230 "cell_type": "markdown",231 "id": "cb921899",232 "metadata": {233 "papermill": {234 "duration": 0.009533,235 "end_time": "2023-02-16T14:31:41.245168",236 "exception": false,237 "start_time": "2023-02-16T14:31:41.235635",238 "status": "completed"239 },240 "tags": []241 },242 "source": [243 "<img src=\"https://i.imgur.com/gb6B4ig.png\" width=\"400\" alt=\"Weights & Biases\" />\n",244 "\n",245 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\"> Weights & Biases (W&B) is a set of machine learning tools that helps you build better models faster. <strong>Kaggle competitions require fast-paced model development and evaluation</strong>. There are a lot of components: exploring the training data, training different models, combining trained models in different combinations (ensembling), and so on.</span>\n",246 "\n",247 "> <span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">⏳ Lots of components = Lots of places to go wrong = Lots of time spent debugging</span>\n",248 "\n",249 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">W&B can be useful for Kaggle competition with it's lightweight and interoperable tools:</span>\n",250 "\n",251 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">To learn more about Weights and Biases check out this <strong><a href=\"https://www.kaggle.com/ayuraj/experiment-tracking-with-weights-and-biases\">kernel</a></strong>.</span>"252 ]253 },254 {255 "cell_type": "code",256 "execution_count": 3,257 "id": "cf7a2ec7",258 "metadata": {259 "execution": {260 "iopub.execute_input": "2023-02-16T14:31:41.264970Z",261 "iopub.status.busy": "2023-02-16T14:31:41.263962Z",262 "iopub.status.idle": "2023-02-16T14:31:44.300089Z",263 "shell.execute_reply": "2023-02-16T14:31:44.298958Z"264 },265 "papermill": {266 "duration": 3.048201,267 "end_time": "2023-02-16T14:31:44.302465",268 "exception": false,269 "start_time": "2023-02-16T14:31:41.254264",270 "status": "completed"271 },272 "tags": []273 },274 "outputs": [275 {276 "name": "stderr",277 "output_type": "stream",278 "text": [279 "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",280 "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",281 "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",282 "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"283 ]284 }285 ],286 "source": [287 "import wandb\n",288 "\n",289 "try:\n",290 " from kaggle_secrets import UserSecretsClient\n",291 " user_secrets = UserSecretsClient()\n",292 " api_key = user_secrets.get_secret(\"wandb_api\")\n",293 " wandb.login(key=api_key)\n",294 " anony = None\n",295 "except:\n",296 " anony = \"must\"\n",297 " print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')"298 ]299 },300 {301 "cell_type": "markdown",302 "id": "185bad4b",303 "metadata": {304 "papermill": {305 "duration": 0.008461,306 "end_time": "2023-02-16T14:31:44.319838",307 "exception": false,308 "start_time": "2023-02-16T14:31:44.311377",309 "status": "completed"310 },311 "tags": []312 },313 "source": [314 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Training Configuration ⚙️</h1></span>"315 ]316 },317 {318 "cell_type": "code",319 "execution_count": 4,320 "id": "884c4ee0",321 "metadata": {322 "execution": {323 "iopub.execute_input": "2023-02-16T14:31:44.338851Z",324 "iopub.status.busy": "2023-02-16T14:31:44.337985Z",325 "iopub.status.idle": "2023-02-16T14:31:48.784003Z",326 "shell.execute_reply": "2023-02-16T14:31:48.782990Z"327 },328 "papermill": {329 "duration": 4.458692,330 "end_time": "2023-02-16T14:31:48.786816",331 "exception": false,332 "start_time": "2023-02-16T14:31:44.328124",333 "status": "completed"334 },335 "tags": []336 },337 "outputs": [338 {339 "data": {340 "application/vnd.jupyter.widget-view+json": {341 "model_id": "f34f39ee4716440abc8031ee3d91f750",342 "version_major": 2,343 "version_minor": 0344 },345 "text/plain": [346 "Downloading (…)rocessor_config.json: 0%| | 0.00/287 [00:00<?, ?B/s]"347 ]348 },349 "metadata": {},350 "output_type": "display_data"351 },352 {353 "data": {354 "application/vnd.jupyter.widget-view+json": {355 "model_id": "93f117668c4746eab3b772fe2810267a",356 "version_major": 2,357 "version_minor": 0358 },359 "text/plain": [360 "Downloading (…)okenizer_config.json: 0%| | 0.00/438 [00:00<?, ?B/s]"361 ]362 },363 "metadata": {},364 "output_type": "display_data"365 },366 {367 "data": {368 "application/vnd.jupyter.widget-view+json": {369 "model_id": "7f23bca725f94dbe8af2a8a856a517f0",370 "version_major": 2,371 "version_minor": 0372 },373 "text/plain": [374 "Downloading (…)solve/main/vocab.txt: 0%| | 0.00/232k [00:00<?, ?B/s]"375 ]376 },377 "metadata": {},378 "output_type": "display_data"379 },380 {381 "data": {382 "application/vnd.jupyter.widget-view+json": {383 "model_id": "8f41215eacf04146a1ada153d0dd589f",384 "version_major": 2,385 "version_minor": 0386 },387 "text/plain": [388 "Downloading (…)cial_tokens_map.json: 0%| | 0.00/125 [00:00<?, ?B/s]"389 ]390 },391 "metadata": {},392 "output_type": "display_data"393 }394 ],395 "source": [396 "CONFIG = {\"seed\": 2023,\n",397 " \"epochs\": 5,\n",398 " \"model_name\": \"Salesforce/blip-image-captioning-base\",\n",399 " \"train_batch_size\": 4,\n",400 " \"valid_batch_size\": 8,\n",401 " \"learning_rate\": 1e-4,\n",402 " \"scheduler\": 'CosineAnnealingLR',\n",403 " \"min_lr\": 1e-6,\n",404 " \"T_max\": 500,\n",405 " \"weight_decay\": 1e-6,\n",406 " \"n_accumulate\": 1,\n",407 " \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n",408 " \"competition\": \"SD\",\n",409 " \"_wandb_kernel\": \"deb\",\n",410 " }\n",411 "\n",412 "CONFIG[\"processor\"] = AutoProcessor.from_pretrained(CONFIG['model_name'])"413 ]414 },415 {416 "cell_type": "markdown",417 "id": "363b03a0",418 "metadata": {419 "papermill": {420 "duration": 0.008772,421 "end_time": "2023-02-16T14:31:48.805153",422 "exception": false,423 "start_time": "2023-02-16T14:31:48.796381",424 "status": "completed"425 },426 "tags": []427 },428 "source": [429 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Set Seed for Reproducibility</h1></span>"430 ]431 },432 {433 "cell_type": "code",434 "execution_count": 5,435 "id": "e25445a2",436 "metadata": {437 "execution": {438 "iopub.execute_input": "2023-02-16T14:31:48.824340Z",439 "iopub.status.busy": "2023-02-16T14:31:48.824010Z",440 "iopub.status.idle": "2023-02-16T14:31:48.833530Z",441 "shell.execute_reply": "2023-02-16T14:31:48.832684Z"442 },443 "papermill": {444 "duration": 0.021404,445 "end_time": "2023-02-16T14:31:48.835520",446 "exception": false,447 "start_time": "2023-02-16T14:31:48.814116",448 "status": "completed"449 },450 "tags": []451 },452 "outputs": [],453 "source": [454 "def set_seed(seed=42):\n",455 " '''Sets the seed of the entire notebook so results are the same every time we run.\n",456 " This is for REPRODUCIBILITY.'''\n",457 " np.random.seed(seed)\n",458 " torch.manual_seed(seed)\n",459 " torch.cuda.manual_seed(seed)\n",460 " # When running on the CuDNN backend, two further options must be set\n",461 " torch.backends.cudnn.deterministic = True\n",462 " torch.backends.cudnn.benchmark = False\n",463 " # Set a fixed value for the hash seed\n",464 " os.environ['PYTHONHASHSEED'] = str(seed)\n",465 " \n",466 "set_seed(CONFIG['seed'])"467 ]468 },469 {470 "cell_type": "markdown",471 "id": "bda70cfe",472 "metadata": {473 "papermill": {474 "duration": 0.008652,475 "end_time": "2023-02-16T14:31:48.853009",476 "exception": false,477 "start_time": "2023-02-16T14:31:48.844357",478 "status": "completed"479 },480 "tags": []481 },482 "source": [483 "# <h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Data 📖</h1>"484 ]485 },486 {487 "cell_type": "markdown",488 "id": "7c95dedb",489 "metadata": {490 "papermill": {491 "duration": 0.008935,492 "end_time": "2023-02-16T14:31:48.870799",493 "exception": false,494 "start_time": "2023-02-16T14:31:48.861864",495 "status": "completed"496 },497 "tags": []498 },499 "source": [500 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">DiffusionDB is the first large-scale text-to-image prompt dataset. It contains 14 million images generated by Stable Diffusion using prompts and hyperparameters specified by real users.</span>\n",501 "<br>\n",502 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">DiffusionDB is publicly available at <a href=\"https://huggingface.co/datasets/poloclub/diffusiondb\">Hugging Face Dataset</a>.</span>\n",503 "<br><hr>\n",504 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">We will use the first 5k images of DiffusionDB-2M subset</span>"505 ]506 },507 {508 "cell_type": "code",509 "execution_count": 6,510 "id": "1f7739ed",511 "metadata": {512 "execution": {513 "iopub.execute_input": "2023-02-16T14:31:48.890138Z",514 "iopub.status.busy": "2023-02-16T14:31:48.889818Z",515 "iopub.status.idle": "2023-02-16T14:35:48.514412Z",516 "shell.execute_reply": "2023-02-16T14:35:48.513429Z"517 },518 "papermill": {519 "duration": 239.63723,520 "end_time": "2023-02-16T14:35:48.516961",521 "exception": false,522 "start_time": "2023-02-16T14:31:48.879731",523 "status": "completed"524 },525 "tags": []526 },527 "outputs": [528 {529 "data": {530 "application/vnd.jupyter.widget-view+json": {531 "model_id": "c7458027dd4843528026ed2309117a47",532 "version_major": 2,533 "version_minor": 0534 },535 "text/plain": [536 "Downloading builder script: 0%| | 0.00/15.0k [00:00<?, ?B/s]"537 ]538 },539 "metadata": {},540 "output_type": "display_data"541 },542 {543 "name": "stdout",544 "output_type": "stream",545 "text": [546 "Downloading and preparing dataset diffusion_db/2m_first_5k to /root/.cache/huggingface/datasets/poloclub___diffusion_db/2m_first_5k/0.9.1/547894e3a57aa647ead68c9faf148324098f47f2bc1ab6705d670721de9d89d1...\n"547 ]548 },549 {550 "data": {551 "application/vnd.jupyter.widget-view+json": {552 "model_id": "1d01ec517a5a43788c5b2ac0f7c4d8d3",553 "version_major": 2,554 "version_minor": 0555 },556 "text/plain": [557 "Downloading data: 0%| | 0.00/581M [00:00<?, ?B/s]"558 ]559 },560 "metadata": {},561 "output_type": "display_data"562 },563 {564 "data": {565 "application/vnd.jupyter.widget-view+json": {566 "model_id": "839e2375745545388fe2db6cd40e09c0",567 "version_major": 2,568 "version_minor": 0569 },570 "text/plain": [571 "Downloading data: 0%| | 0.00/585M [00:00<?, ?B/s]"572 ]573 },574 "metadata": {},575 "output_type": "display_data"576 },577 {578 "data": {579 "application/vnd.jupyter.widget-view+json": {580 "model_id": "84d62e6d749849f2b730aed10ce09008",581 "version_major": 2,582 "version_minor": 0583 },584 "text/plain": [585 "Downloading data: 0%| | 0.00/643M [00:00<?, ?B/s]"586 ]587 },588 "metadata": {},589 "output_type": "display_data"590 },591 {592 "data": {593 "application/vnd.jupyter.widget-view+json": {594 "model_id": "67260f48c5e74cd2a61d1dc6ae7ca535",595 "version_major": 2,596 "version_minor": 0597 },598 "text/plain": [599 "Downloading data: 0%| | 0.00/585M [00:00<?, ?B/s]"600 ]601 },602 "metadata": {},603 "output_type": "display_data"604 },605 {606 "data": {607 "application/vnd.jupyter.widget-view+json": {608 "model_id": "e41c62969c7c492791fffb7ac90e4062",609 "version_major": 2,610 "version_minor": 0611 },612 "text/plain": [613 "Downloading data: 0%| | 0.00/595M [00:00<?, ?B/s]"614 ]615 },616 "metadata": {},617 "output_type": "display_data"618 },619 {620 "data": {621 "application/vnd.jupyter.widget-view+json": {622 "model_id": "47529ba61bc3496783ca914cddbe2243",623 "version_major": 2,624 "version_minor": 0625 },626 "text/plain": [627 "Downloading data: 0%| | 0.00/195M [00:00<?, ?B/s]"628 ]629 },630 "metadata": {},631 "output_type": "display_data"632 },633 {634 "data": {635 "application/vnd.jupyter.widget-view+json": {636 "model_id": "f86ea5208e8a44a5b97f8a5524d15cf3",637 "version_major": 2,638 "version_minor": 0639 },640 "text/plain": [641 "Generating train split: 0 examples [00:00, ? examples/s]"642 ]643 },644 "metadata": {},645 "output_type": "display_data"646 },647 {648 "name": "stdout",649 "output_type": "stream",650 "text": [651 "Dataset diffusion_db downloaded and prepared to /root/.cache/huggingface/datasets/poloclub___diffusion_db/2m_first_5k/0.9.1/547894e3a57aa647ead68c9faf148324098f47f2bc1ab6705d670721de9d89d1. Subsequent calls will reuse this data.\n"652 ]653 },654 {655 "data": {656 "application/vnd.jupyter.widget-view+json": {657 "model_id": "6975a073a0234b45a42ad533bbf89818",658 "version_major": 2,659 "version_minor": 0660 },661 "text/plain": [662 " 0%| | 0/1 [00:00<?, ?it/s]"663 ]664 },665 "metadata": {},666 "output_type": "display_data"667 }668 ],669 "source": [670 "from datasets import load_dataset\n",671 "\n",672 "# Load the dataset with the `2m_first_5k` subset\n",673 "dataset = load_dataset('poloclub/diffusiondb', '2m_first_5k')"674 ]675 },676 {677 "cell_type": "code",678 "execution_count": 7,679 "id": "cbeb0e3c",680 "metadata": {681 "execution": {682 "iopub.execute_input": "2023-02-16T14:35:48.540211Z",683 "iopub.status.busy": "2023-02-16T14:35:48.539594Z",684 "iopub.status.idle": "2023-02-16T14:35:48.548663Z",685 "shell.execute_reply": "2023-02-16T14:35:48.547676Z"686 },687 "papermill": {688 "duration": 0.022645,689 "end_time": "2023-02-16T14:35:48.550732",690 "exception": false,691 "start_time": "2023-02-16T14:35:48.528087",692 "status": "completed"693 },694 "tags": []695 },696 "outputs": [697 {698 "data": {699 "text/plain": [700 "DatasetDict({\n",701 " train: Dataset({\n",702 " features: ['image', 'prompt', 'seed', 'step', 'cfg', 'sampler', 'width', 'height', 'user_name', 'timestamp', 'image_nsfw', 'prompt_nsfw'],\n",703 " num_rows: 5000\n",704 " })\n",705 "})"706 ]707 },708 "execution_count": 7,709 "metadata": {},710 "output_type": "execute_result"711 }712 ],713 "source": [714 "dataset"715 ]716 },717 {718 "cell_type": "code",719 "execution_count": 8,720 "id": "909582fd",721 "metadata": {722 "execution": {723 "iopub.execute_input": "2023-02-16T14:35:48.573250Z",724 "iopub.status.busy": "2023-02-16T14:35:48.572975Z",725 "iopub.status.idle": "2023-02-16T14:36:54.214347Z",726 "shell.execute_reply": "2023-02-16T14:36:54.213414Z"727 },728 "papermill": {729 "duration": 65.654845,730 "end_time": "2023-02-16T14:36:54.216482",731 "exception": false,732 "start_time": "2023-02-16T14:35:48.561637",733 "status": "completed"734 },735 "tags": []736 },737 "outputs": [738 {739 "data": {740 "application/vnd.jupyter.widget-view+json": {741 "model_id": "9858217353a2402e96b5187fb79da8f4",742 "version_major": 2,743 "version_minor": 0744 },745 "text/plain": [746 " 0%| | 0/5 [00:00<?, ?ba/s]"747 ]748 },749 "metadata": {},750 "output_type": "display_data"751 },752 {753 "data": {754 "text/plain": [755 "4984"756 ]757 },758 "execution_count": 8,759 "metadata": {},760 "output_type": "execute_result"761 }762 ],763 "source": [764 "dataset = dataset['train']\n",765 "dataset = dataset.filter(lambda example: example[\"step\"] == 50)\n",766 "len(dataset)"767 ]768 },769 {770 "cell_type": "code",771 "execution_count": 9,772 "id": "d0095214",773 "metadata": {774 "execution": {775 "iopub.execute_input": "2023-02-16T14:36:54.239359Z",776 "iopub.status.busy": "2023-02-16T14:36:54.239065Z",777 "iopub.status.idle": "2023-02-16T14:36:54.259184Z",778 "shell.execute_reply": "2023-02-16T14:36:54.258135Z"779 },780 "papermill": {781 "duration": 0.033815,782 "end_time": "2023-02-16T14:36:54.261419",783 "exception": false,784 "start_time": "2023-02-16T14:36:54.227604",785 "status": "completed"786 },787 "tags": []788 },789 "outputs": [790 {791 "data": {792 "text/plain": [793 "{'image': <PIL.PngImagePlugin.PngImageFile image mode=RGB size=512x768>,\n",794 " 'prompt': 'a renaissance portrait of dwayne johnson, art in the style of rembrandt!! intricate. ultra detailed, oil on canvas, wet - on - wet technique, pay attention to facial details, highly realistic, cinematic lightning, intricate textures, illusionistic detail, ',\n",795 " 'seed': 2480545905,\n",796 " 'step': 50,\n",797 " 'cfg': 16.0,\n",798 " 'sampler': 'k_euler_ancestral',\n",799 " 'width': 512,\n",800 " 'height': 768,\n",801 " 'user_name': 'e9dfc969d22cb9c5621ad075b3826c28f18ef3840c6dda59c4ac7daa55241393',\n",802 " 'timestamp': datetime.datetime(2022, 8, 20, 5, 28, tzinfo=<UTC>),\n",803 " 'image_nsfw': 0.16348764300346375,\n",804 " 'prompt_nsfw': 0.000792665290646255}"805 ]806 },807 "execution_count": 9,808 "metadata": {},809 "output_type": "execute_result"810 }811 ],812 "source": [813 "dataset[0]"814 ]815 },816 {817 "cell_type": "markdown",818 "id": "51ae43dc",819 "metadata": {820 "papermill": {821 "duration": 0.010188,822 "end_time": "2023-02-16T14:36:54.282152",823 "exception": false,824 "start_time": "2023-02-16T14:36:54.271964",825 "status": "completed"826 },827 "tags": []828 },829 "source": [830 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Data Split</h1></span>"831 ]832 },833 {834 "cell_type": "code",835 "execution_count": 10,836 "id": "1d8beb86",837 "metadata": {838 "execution": {839 "iopub.execute_input": "2023-02-16T14:36:54.304302Z",840 "iopub.status.busy": "2023-02-16T14:36:54.303758Z",841 "iopub.status.idle": "2023-02-16T14:36:54.333924Z",842 "shell.execute_reply": "2023-02-16T14:36:54.333072Z"843 },844 "papermill": {845 "duration": 0.043385,846 "end_time": "2023-02-16T14:36:54.335921",847 "exception": false,848 "start_time": "2023-02-16T14:36:54.292536",849 "status": "completed"850 },851 "tags": []852 },853 "outputs": [],854 "source": [855 "dataset = dataset.train_test_split(test_size=0.1)"856 ]857 },858 {859 "cell_type": "markdown",860 "id": "18edf601",861 "metadata": {862 "papermill": {863 "duration": 0.010211,864 "end_time": "2023-02-16T14:36:54.356668",865 "exception": false,866 "start_time": "2023-02-16T14:36:54.346457",867 "status": "completed"868 },869 "tags": []870 },871 "source": [872 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Dataset Class</h1></span>"873 ]874 },875 {876 "cell_type": "code",877 "execution_count": 11,878 "id": "273d9089",879 "metadata": {880 "execution": {881 "iopub.execute_input": "2023-02-16T14:36:54.379003Z",882 "iopub.status.busy": "2023-02-16T14:36:54.378353Z",883 "iopub.status.idle": "2023-02-16T14:36:54.384519Z",884 "shell.execute_reply": "2023-02-16T14:36:54.383628Z"885 },886 "papermill": {887 "duration": 0.019325,888 "end_time": "2023-02-16T14:36:54.386462",889 "exception": false,890 "start_time": "2023-02-16T14:36:54.367137",891 "status": "completed"892 },893 "tags": []894 },895 "outputs": [],896 "source": [897 "class ImageCaptioningDataset(Dataset):\n",898 " def __init__(self, dataset, processor):\n",899 " self.dataset = dataset\n",900 " self.processor = processor\n",901 "\n",902 " def __len__(self):\n",903 " return len(self.dataset)\n",904 "\n",905 " def __getitem__(self, idx):\n",906 " item = self.dataset[idx]\n",907 " encoding = self.processor(images=item[\"image\"], text=item[\"prompt\"], \n",908 " padding=\"max_length\", return_tensors=\"pt\")\n",909 " # remove batch dimension\n",910 " encoding = {k:v.squeeze() for k,v in encoding.items()}\n",911 " return encoding"912 ]913 },914 {915 "cell_type": "code",916 "execution_count": 12,917 "id": "08092c0e",918 "metadata": {919 "execution": {920 "iopub.execute_input": "2023-02-16T14:36:54.408727Z",921 "iopub.status.busy": "2023-02-16T14:36:54.408050Z",922 "iopub.status.idle": "2023-02-16T14:36:54.412524Z",923 "shell.execute_reply": "2023-02-16T14:36:54.411632Z"924 },925 "papermill": {926 "duration": 0.017727,927 "end_time": "2023-02-16T14:36:54.414533",928 "exception": false,929 "start_time": "2023-02-16T14:36:54.396806",930 "status": "completed"931 },932 "tags": []933 },934 "outputs": [],935 "source": [936 "train_dataset = ImageCaptioningDataset(dataset['train'], CONFIG['processor'])\n",937 "valid_dataset = ImageCaptioningDataset(dataset['test'], CONFIG['processor'])"938 ]939 },940 {941 "cell_type": "code",942 "execution_count": 13,943 "id": "62e3efa2",944 "metadata": {945 "execution": {946 "iopub.execute_input": "2023-02-16T14:36:54.436815Z",947 "iopub.status.busy": "2023-02-16T14:36:54.436048Z",948 "iopub.status.idle": "2023-02-16T14:36:54.593437Z",949 "shell.execute_reply": "2023-02-16T14:36:54.592283Z"950 },951 "papermill": {952 "duration": 0.171213,953 "end_time": "2023-02-16T14:36:54.596119",954 "exception": false,955 "start_time": "2023-02-16T14:36:54.424906",956 "status": "completed"957 },958 "tags": []959 },960 "outputs": [961 {962 "data": {963 "text/plain": [964 "dict_keys(['pixel_values', 'input_ids', 'attention_mask'])"965 ]966 },967 "execution_count": 13,968 "metadata": {},969 "output_type": "execute_result"970 }971 ],972 "source": [973 "train_dataset[0].keys()"974 ]975 },976 {977 "cell_type": "markdown",978 "id": "cff8fd35",979 "metadata": {980 "papermill": {981 "duration": 0.010438,982 "end_time": "2023-02-16T14:36:54.617520",983 "exception": false,984 "start_time": "2023-02-16T14:36:54.607082",985 "status": "completed"986 },987 "tags": []988 },989 "source": [990 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Model</h1></span>"991 ]992 },993 {994 "cell_type": "markdown",995 "id": "bbc5df20",996 "metadata": {997 "papermill": {998 "duration": 0.010408,999 "end_time": "2023-02-16T14:36:54.638512",1000 "exception": false,1001 "start_time": "2023-02-16T14:36:54.628104",1002 "status": "completed"1003 },1004 "tags": []1005 },1006 "source": [1007 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">BLIP is a model that is able to perform various multi-modal tasks including Image captioning </span> <br>\n",1008 "<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">Model documentation: https://huggingface.co/docs/transformers/model_doc/blip </span>"1009 ]1010 },1011 {1012 "cell_type": "code",1013 "execution_count": 14,1014 "id": "a7d712a3",1015 "metadata": {1016 "execution": {1017 "iopub.execute_input": "2023-02-16T14:36:54.661659Z",1018 "iopub.status.busy": "2023-02-16T14:36:54.660654Z",1019 "iopub.status.idle": "2023-02-16T14:38:23.012450Z",1020 "shell.execute_reply": "2023-02-16T14:38:23.011316Z"1021 },1022 "papermill": {1023 "duration": 88.366068,1024 "end_time": "2023-02-16T14:38:23.015065",1025 "exception": false,1026 "start_time": "2023-02-16T14:36:54.648997",1027 "status": "completed"1028 },1029 "tags": []1030 },1031 "outputs": [1032 {1033 "data": {1034 "application/vnd.jupyter.widget-view+json": {1035 "model_id": "c913bc7f29b54b8fa83c8736e7444b5b",1036 "version_major": 2,1037 "version_minor": 01038 },1039 "text/plain": [1040 "Downloading (…)lve/main/config.json: 0%| | 0.00/4.56k [00:00<?, ?B/s]"1041 ]1042 },1043 "metadata": {},1044 "output_type": "display_data"1045 },1046 {1047 "data": {1048 "application/vnd.jupyter.widget-view+json": {1049 "model_id": "ed08c794b8a045079f326439c7db9c37",1050 "version_major": 2,1051 "version_minor": 01052 },1053 "text/plain": [1054 "Downloading (…)\"pytorch_model.bin\";: 0%| | 0.00/990M [00:00<?, ?B/s]"1055 ]1056 },1057 "metadata": {},1058 "output_type": "display_data"1059 }1060 ],1061 "source": [1062 "model = BlipForConditionalGeneration.from_pretrained(CONFIG['model_name'])"1063 ]1064 },1065 {1066 "cell_type": "markdown",1067 "id": "db59d5fd",1068 "metadata": {1069 "papermill": {1070 "duration": 0.011447,1071 "end_time": "2023-02-16T14:38:23.038213",1072 "exception": false,1073 "start_time": "2023-02-16T14:38:23.026766",1074 "status": "completed"1075 },1076 "tags": []1077 },1078 "source": [1079 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Training Function</h1></span>"1080 ]1081 },1082 {1083 "cell_type": "code",1084 "execution_count": 15,1085 "id": "c9739d37",1086 "metadata": {1087 "execution": {1088 "iopub.execute_input": "2023-02-16T14:38:23.062685Z",1089 "iopub.status.busy": "2023-02-16T14:38:23.061656Z",1090 "iopub.status.idle": "2023-02-16T14:38:23.070634Z",1091 "shell.execute_reply": "2023-02-16T14:38:23.069686Z"1092 },1093 "papermill": {1094 "duration": 0.023773,1095 "end_time": "2023-02-16T14:38:23.072976",1096 "exception": false,1097 "start_time": "2023-02-16T14:38:23.049203",1098 "status": "completed"1099 },1100 "tags": []1101 },1102 "outputs": [],1103 "source": [1104 "def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n",1105 " model.train()\n",1106 " \n",1107 " dataset_size = 0\n",1108 " running_loss = 0.0\n",1109 " \n",1110 " bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",1111 " for step, data in bar:\n",1112 " input_ids = data['input_ids'].to(device)\n",1113 " pixel_values = data['pixel_values'].to(device)\n",1114 " \n",1115 " batch_size = input_ids.size(0)\n",1116 "\n",1117 " outputs = model(input_ids=input_ids, \n",1118 " pixel_values=pixel_values, \n",1119 " labels=input_ids)\n",1120 " \n",1121 " loss = outputs.loss\n",1122 " loss = loss / CONFIG['n_accumulate']\n",1123 " loss.backward()\n",1124 " \n",1125 " if (step + 1) % CONFIG['n_accumulate'] == 0:\n",1126 " optimizer.step()\n",1127 "\n",1128 " # zero the parameter gradients\n",1129 " optimizer.zero_grad()\n",1130 "\n",1131 " if scheduler is not None:\n",1132 " scheduler.step()\n",1133 " \n",1134 " running_loss += (loss.item() * batch_size)\n",1135 " dataset_size += batch_size\n",1136 " \n",1137 " epoch_loss = running_loss / dataset_size\n",1138 " \n",1139 " bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n",1140 " LR=optimizer.param_groups[0]['lr'])\n",1141 " gc.collect()\n",1142 " \n",1143 " return epoch_loss"1144 ]1145 },1146 {1147 "cell_type": "markdown",1148 "id": "f3f121ad",1149 "metadata": {1150 "papermill": {1151 "duration": 0.012242,1152 "end_time": "2023-02-16T14:38:23.096437",1153 "exception": false,1154 "start_time": "2023-02-16T14:38:23.084195",1155 "status": "completed"1156 },1157 "tags": []1158 },1159 "source": [1160 "# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Validation Function</h1></span>"1161 ]1162 },1163 {1164 "cell_type": "code",1165 "execution_count": 16,1166 "id": "72d7eb0a",1167 "metadata": {1168 "execution": {1169 "iopub.execute_input": "2023-02-16T14:38:23.119485Z",1170 "iopub.status.busy": "2023-02-16T14:38:23.119194Z",1171 "iopub.status.idle": "2023-02-16T14:38:23.126580Z",1172 "shell.execute_reply": "2023-02-16T14:38:23.125520Z"1173 },1174 "papermill": {1175 "duration": 0.021512,1176 "end_time": "2023-02-16T14:38:23.128797",1177 "exception": false,1178 "start_time": "2023-02-16T14:38:23.107285",1179 "status": "completed"1180 },1181 "tags": []1182 },1183 "outputs": [],1184 "source": [1185 "@torch.no_grad()\n",1186 "def valid_one_epoch(model, dataloader, device, epoch):\n",1187 " model.eval()\n",1188 " \n",1189 " dataset_size = 0\n",1190 " running_loss = 0.0\n",1191 " \n",1192 " bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",1193 " for step, data in bar: \n",1194 " input_ids = data['input_ids'].to(device)\n",1195 " pixel_values = data['pixel_values'].to(device)\n",1196 " \n",1197 " batch_size = input_ids.size(0)\n",1198 "\n",1199 " outputs = model(input_ids=input_ids, \n",1200 " pixel_values=pixel_values, \n",