PEFT/conditional-generation
4
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 12,6 "metadata": {7 "id": "5f93b7d1"8 },9 "outputs": [],10 "source": [11 "from transformers import AutoModelForSeq2SeqLM\n",12 "import peft\n",13 "from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, IA3Config, TaskType\n",14 "import torch\n",15 "from datasets import load_dataset\n",16 "import os\n",17 "\n",18 "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",19 "from transformers import AutoTokenizer\n",20 "from torch.utils.data import DataLoader\n",21 "from transformers import default_data_collator, get_linear_schedule_with_warmup\n",22 "from tqdm import tqdm\n",23 "from datasets import load_dataset\n",24 "\n",25 "device = \"cuda\"\n",26 "model_name_or_path = \"bigscience/mt0-large\"\n",27 "tokenizer_name_or_path = \"bigscience/mt0-large\"\n",28 "\n",29 "checkpoint_name = \"financial_sentiment_analysis_ia3_v1.pt\"\n",30 "text_column = \"sentence\"\n",31 "label_column = \"text_label\"\n",32 "max_length = 128\n",33 "lr = 8e-3\n",34 "num_epochs = 3\n",35 "batch_size = 8"36 ]37 },38 {39 "cell_type": "code",40 "execution_count": 13,41 "metadata": {42 "colab": {43 "base_uri": "https://localhost:8080/"44 },45 "id": "b9e6368c",46 "outputId": "fc2888a8-4fe9-4d61-dd2d-753e751e1416"47 },48 "outputs": [49 {50 "data": {51 "text/plain": [52 "<module 'peft' from '/usr/local/lib/python3.10/dist-packages/peft/__init__.py'>"53 ]54 },55 "execution_count": 13,56 "metadata": {},57 "output_type": "execute_result"58 }59 ],60 "source": [61 "import importlib\n",62 "\n",63 "importlib.reload(peft)"64 ]65 },66 {67 "cell_type": "code",68 "execution_count": 14,69 "metadata": {70 "id": "8d0850ac"71 },72 "outputs": [],73 "source": [74 "# creating model\n",75 "peft_config = IA3Config(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, feedforward_modules=[])\n",76 "\n",77 "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)"78 ]79 },80 {81 "cell_type": "code",82 "execution_count": 15,83 "metadata": {84 "colab": {85 "base_uri": "https://localhost:8080/"86 },87 "id": "e10c3831",88 "outputId": "e69c5e07-ae58-446c-8301-e99ac6b85d62"89 },90 "outputs": [91 {92 "data": {93 "text/plain": [94 "MT5ForConditionalGeneration(\n",95 " (shared): Embedding(250112, 1024)\n",96 " (encoder): MT5Stack(\n",97 " (embed_tokens): Embedding(250112, 1024)\n",98 " (block): ModuleList(\n",99 " (0): MT5Block(\n",100 " (layer): ModuleList(\n",101 " (0): MT5LayerSelfAttention(\n",102 " (SelfAttention): MT5Attention(\n",103 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",104 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",105 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",106 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",107 " (relative_attention_bias): Embedding(32, 16)\n",108 " )\n",109 " (layer_norm): MT5LayerNorm()\n",110 " (dropout): Dropout(p=0.1, inplace=False)\n",111 " )\n",112 " (1): MT5LayerFF(\n",113 " (DenseReluDense): MT5DenseGatedActDense(\n",114 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",115 " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n",116 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",117 " (dropout): Dropout(p=0.1, inplace=False)\n",118 " (act): NewGELUActivation()\n",119 " )\n",120 " (layer_norm): MT5LayerNorm()\n",121 " (dropout): Dropout(p=0.1, inplace=False)\n",122 " )\n",123 " )\n",124 " )\n",125 " (1-23): 23 x MT5Block(\n",126 " (layer): ModuleList(\n",127 " (0): MT5LayerSelfAttention(\n",128 " (SelfAttention): MT5Attention(\n",129 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",130 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",131 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",132 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",133 " )\n",134 " (layer_norm): MT5LayerNorm()\n",135 " (dropout): Dropout(p=0.1, inplace=False)\n",136 " )\n",137 " (1): MT5LayerFF(\n",138 " (DenseReluDense): MT5DenseGatedActDense(\n",139 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",140 " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n",141 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",142 " (dropout): Dropout(p=0.1, inplace=False)\n",143 " (act): NewGELUActivation()\n",144 " )\n",145 " (layer_norm): MT5LayerNorm()\n",146 " (dropout): Dropout(p=0.1, inplace=False)\n",147 " )\n",148 " )\n",149 " )\n",150 " )\n",151 " (final_layer_norm): MT5LayerNorm()\n",152 " (dropout): Dropout(p=0.1, inplace=False)\n",153 " )\n",154 " (decoder): MT5Stack(\n",155 " (embed_tokens): Embedding(250112, 1024)\n",156 " (block): ModuleList(\n",157 " (0): MT5Block(\n",158 " (layer): ModuleList(\n",159 " (0): MT5LayerSelfAttention(\n",160 " (SelfAttention): MT5Attention(\n",161 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",162 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",163 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",164 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",165 " (relative_attention_bias): Embedding(32, 16)\n",166 " )\n",167 " (layer_norm): MT5LayerNorm()\n",168 " (dropout): Dropout(p=0.1, inplace=False)\n",169 " )\n",170 " (1): MT5LayerCrossAttention(\n",171 " (EncDecAttention): MT5Attention(\n",172 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",173 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",174 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",175 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",176 " )\n",177 " (layer_norm): MT5LayerNorm()\n",178 " (dropout): Dropout(p=0.1, inplace=False)\n",179 " )\n",180 " (2): MT5LayerFF(\n",181 " (DenseReluDense): MT5DenseGatedActDense(\n",182 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",183 " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n",184 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",185 " (dropout): Dropout(p=0.1, inplace=False)\n",186 " (act): NewGELUActivation()\n",187 " )\n",188 " (layer_norm): MT5LayerNorm()\n",189 " (dropout): Dropout(p=0.1, inplace=False)\n",190 " )\n",191 " )\n",192 " )\n",193 " (1-23): 23 x MT5Block(\n",194 " (layer): ModuleList(\n",195 " (0): MT5LayerSelfAttention(\n",196 " (SelfAttention): MT5Attention(\n",197 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",198 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",199 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",200 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",201 " )\n",202 " (layer_norm): MT5LayerNorm()\n",203 " (dropout): Dropout(p=0.1, inplace=False)\n",204 " )\n",205 " (1): MT5LayerCrossAttention(\n",206 " (EncDecAttention): MT5Attention(\n",207 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",208 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",209 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",210 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",211 " )\n",212 " (layer_norm): MT5LayerNorm()\n",213 " (dropout): Dropout(p=0.1, inplace=False)\n",214 " )\n",215 " (2): MT5LayerFF(\n",216 " (DenseReluDense): MT5DenseGatedActDense(\n",217 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",218 " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n",219 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",220 " (dropout): Dropout(p=0.1, inplace=False)\n",221 " (act): NewGELUActivation()\n",222 " )\n",223 " (layer_norm): MT5LayerNorm()\n",224 " (dropout): Dropout(p=0.1, inplace=False)\n",225 " )\n",226 " )\n",227 " )\n",228 " )\n",229 " (final_layer_norm): MT5LayerNorm()\n",230 " (dropout): Dropout(p=0.1, inplace=False)\n",231 " )\n",232 " (lm_head): Linear(in_features=1024, out_features=250112, bias=False)\n",233 ")"234 ]235 },236 "execution_count": 15,237 "metadata": {},238 "output_type": "execute_result"239 }240 ],241 "source": [242 "model"243 ]244 },245 {246 "cell_type": "code",247 "execution_count": 16,248 "metadata": {249 "colab": {250 "base_uri": "https://localhost:8080/"251 },252 "id": "05978e96",253 "outputId": "ea9b7d40-010f-4df0-ec64-a7146a5f8b08"254 },255 "outputs": [256 {257 "name": "stdout",258 "output_type": "stream",259 "text": [260 "trainable params: 282,624 || all params: 1,229,863,936 || trainable%: 0.022980103060766553\n"261 ]262 },263 {264 "data": {265 "text/plain": [266 "PeftModelForSeq2SeqLM(\n",267 " (base_model): IA3Model(\n",268 " (model): MT5ForConditionalGeneration(\n",269 " (shared): Embedding(250112, 1024)\n",270 " (encoder): MT5Stack(\n",271 " (embed_tokens): Embedding(250112, 1024)\n",272 " (block): ModuleList(\n",273 " (0): MT5Block(\n",274 " (layer): ModuleList(\n",275 " (0): MT5LayerSelfAttention(\n",276 " (SelfAttention): MT5Attention(\n",277 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",278 " (k): Linear(\n",279 " in_features=1024, out_features=1024, bias=False\n",280 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",281 " )\n",282 " (v): Linear(\n",283 " in_features=1024, out_features=1024, bias=False\n",284 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",285 " )\n",286 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",287 " (relative_attention_bias): Embedding(32, 16)\n",288 " )\n",289 " (layer_norm): MT5LayerNorm()\n",290 " (dropout): Dropout(p=0.1, inplace=False)\n",291 " )\n",292 " (1): MT5LayerFF(\n",293 " (DenseReluDense): MT5DenseGatedActDense(\n",294 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",295 " (wi_1): Linear(\n",296 " in_features=1024, out_features=2816, bias=False\n",297 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 2816x1])\n",298 " )\n",299 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",300 " (dropout): Dropout(p=0.1, inplace=False)\n",301 " (act): NewGELUActivation()\n",302 " )\n",303 " (layer_norm): MT5LayerNorm()\n",304 " (dropout): Dropout(p=0.1, inplace=False)\n",305 " )\n",306 " )\n",307 " )\n",308 " (1-23): 23 x MT5Block(\n",309 " (layer): ModuleList(\n",310 " (0): MT5LayerSelfAttention(\n",311 " (SelfAttention): MT5Attention(\n",312 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",313 " (k): Linear(\n",314 " in_features=1024, out_features=1024, bias=False\n",315 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",316 " )\n",317 " (v): Linear(\n",318 " in_features=1024, out_features=1024, bias=False\n",319 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",320 " )\n",321 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",322 " )\n",323 " (layer_norm): MT5LayerNorm()\n",324 " (dropout): Dropout(p=0.1, inplace=False)\n",325 " )\n",326 " (1): MT5LayerFF(\n",327 " (DenseReluDense): MT5DenseGatedActDense(\n",328 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",329 " (wi_1): Linear(\n",330 " in_features=1024, out_features=2816, bias=False\n",331 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 2816x1])\n",332 " )\n",333 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",334 " (dropout): Dropout(p=0.1, inplace=False)\n",335 " (act): NewGELUActivation()\n",336 " )\n",337 " (layer_norm): MT5LayerNorm()\n",338 " (dropout): Dropout(p=0.1, inplace=False)\n",339 " )\n",340 " )\n",341 " )\n",342 " )\n",343 " (final_layer_norm): MT5LayerNorm()\n",344 " (dropout): Dropout(p=0.1, inplace=False)\n",345 " )\n",346 " (decoder): MT5Stack(\n",347 " (embed_tokens): Embedding(250112, 1024)\n",348 " (block): ModuleList(\n",349 " (0): MT5Block(\n",350 " (layer): ModuleList(\n",351 " (0): MT5LayerSelfAttention(\n",352 " (SelfAttention): MT5Attention(\n",353 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",354 " (k): Linear(\n",355 " in_features=1024, out_features=1024, bias=False\n",356 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",357 " )\n",358 " (v): Linear(\n",359 " in_features=1024, out_features=1024, bias=False\n",360 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",361 " )\n",362 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",363 " (relative_attention_bias): Embedding(32, 16)\n",364 " )\n",365 " (layer_norm): MT5LayerNorm()\n",366 " (dropout): Dropout(p=0.1, inplace=False)\n",367 " )\n",368 " (1): MT5LayerCrossAttention(\n",369 " (EncDecAttention): MT5Attention(\n",370 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",371 " (k): Linear(\n",372 " in_features=1024, out_features=1024, bias=False\n",373 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",374 " )\n",375 " (v): Linear(\n",376 " in_features=1024, out_features=1024, bias=False\n",377 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",378 " )\n",379 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",380 " )\n",381 " (layer_norm): MT5LayerNorm()\n",382 " (dropout): Dropout(p=0.1, inplace=False)\n",383 " )\n",384 " (2): MT5LayerFF(\n",385 " (DenseReluDense): MT5DenseGatedActDense(\n",386 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",387 " (wi_1): Linear(\n",388 " in_features=1024, out_features=2816, bias=False\n",389 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 2816x1])\n",390 " )\n",391 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",392 " (dropout): Dropout(p=0.1, inplace=False)\n",393 " (act): NewGELUActivation()\n",394 " )\n",395 " (layer_norm): MT5LayerNorm()\n",396 " (dropout): Dropout(p=0.1, inplace=False)\n",397 " )\n",398 " )\n",399 " )\n",400 " (1-23): 23 x MT5Block(\n",401 " (layer): ModuleList(\n",402 " (0): MT5LayerSelfAttention(\n",403 " (SelfAttention): MT5Attention(\n",404 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",405 " (k): Linear(\n",406 " in_features=1024, out_features=1024, bias=False\n",407 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",408 " )\n",409 " (v): Linear(\n",410 " in_features=1024, out_features=1024, bias=False\n",411 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",412 " )\n",413 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",414 " )\n",415 " (layer_norm): MT5LayerNorm()\n",416 " (dropout): Dropout(p=0.1, inplace=False)\n",417 " )\n",418 " (1): MT5LayerCrossAttention(\n",419 " (EncDecAttention): MT5Attention(\n",420 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",421 " (k): Linear(\n",422 " in_features=1024, out_features=1024, bias=False\n",423 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",424 " )\n",425 " (v): Linear(\n",426 " in_features=1024, out_features=1024, bias=False\n",427 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",428 " )\n",429 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",430 " )\n",431 " (layer_norm): MT5LayerNorm()\n",432 " (dropout): Dropout(p=0.1, inplace=False)\n",433 " )\n",434 " (2): MT5LayerFF(\n",435 " (DenseReluDense): MT5DenseGatedActDense(\n",436 " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n",437 " (wi_1): Linear(\n",438 " in_features=1024, out_features=2816, bias=False\n",439 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 2816x1])\n",440 " )\n",441 " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n",442 " (dropout): Dropout(p=0.1, inplace=False)\n",443 " (act): NewGELUActivation()\n",444 " )\n",445 " (layer_norm): MT5LayerNorm()\n",446 " (dropout): Dropout(p=0.1, inplace=False)\n",447 " )\n",448 " )\n",449 " )\n",450 " )\n",451 " (final_layer_norm): MT5LayerNorm()\n",452 " (dropout): Dropout(p=0.1, inplace=False)\n",453 " )\n",454 " (lm_head): Linear(in_features=1024, out_features=250112, bias=False)\n",455 " )\n",456 " )\n",457 ")"458 ]459 },460 "execution_count": 16,461 "metadata": {},462 "output_type": "execute_result"463 }464 ],465 "source": [466 "model = get_peft_model(model, peft_config)\n",467 "model.print_trainable_parameters()\n",468 "model"469 ]470 },471 {472 "cell_type": "code",473 "execution_count": 17,474 "metadata": {475 "colab": {476 "base_uri": "https://localhost:8080/",477 "height": 140,478 "referenced_widgets": [479 "bbfb7533b5ca459194e171df56b79566",480 "c894e8237aa34c56bb250acab1466005",481 "a5a126b229064812bf3dcb228118be50",482 "661e1b29c59a4295b594edfa4f50ff87",483 "1bcba805972b484d8b6aa6542c81841c",484 "e71f5c7f1d5d4f83b58c68d2fa310d9c",485 "6a567e0a1a5447519c5df10e777520cf",486 "7aeca19b84904906a04c12659f84ff9e",487 "dd4b895874ce46ceb1ad0d9bc973f98f",488 "b138f91be7f94008806eaf0a6988bc3f",489 "da14180f51ab44b48470cb9ea74d3864",490 "9e12d97af6124a5a8c6627708b300c1e",491 "faa18df899c14e9cac6721253e6c9128",492 "79d0ede7a5b24756aa6d34fda8c29159",493 "3b175b452f4347558aa3c4501cc90030",494 "fc4637a1b37e4e90874c71aa4271ac74",495 "1b8aada826a0451bb60c418b19178c8c",496 "a91916e02e9c424e881e45b3aa978574",497 "ca509bd409624c998e555c9a779b8aae",498 "9c890fc422954347b86d3bde7a421caf",499 "6f9453484ea94587a64d70f1b3a1f6e4",500 "48770ef159f44c01be2a75c75aecd80f",501 "0c561dab67914ea9b6e1aab803600551",502 "1e021a1954b44d69a90101a96c360661",503 "013e3343285f437a893bdd673fb90e22",504 "28802da68fb04d70b1c6bc511a04676f",505 "94174da0d6554be087d4527bea5b511a",506 "dc8ab16a1e6c4e6893c95ccd16568f9a",507 "72383136663448d89cf3b82b87cbb392",508 "5b1bdaf16cbc473081e4237f839167b9",509 "51f8fb45485540bb985b606d43ae04ea",510 "f760cd4758334ca9a43fd15612fd808b",511 "f60e9915d2a74ca7bc010d7684f5acf6"512 ]513 },514 "id": "4ee2babf",515 "outputId": "3c413083-247d-47da-f25c-032764be0beb"516 },517 "outputs": [518 {519 "name": "stderr",520 "output_type": "stream",521 "text": [522 "WARNING:datasets.builder:Found cached dataset financial_phrasebank (/root/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n"523 ]524 },525 {526 "data": {527 "application/vnd.jupyter.widget-view+json": {528 "model_id": "bbfb7533b5ca459194e171df56b79566",529 "version_major": 2,530 "version_minor": 0531 },532 "text/plain": [533 " 0%| | 0/1 [00:00<?, ?it/s]"534 ]535 },536 "metadata": {},537 "output_type": "display_data"538 },539 {540 "data": {541 "application/vnd.jupyter.widget-view+json": {542 "model_id": "9e12d97af6124a5a8c6627708b300c1e",543 "version_major": 2,544 "version_minor": 0545 },546 "text/plain": [547 "Map: 0%| | 0/2037 [00:00<?, ? examples/s]"548 ]549 },550 "metadata": {},551 "output_type": "display_data"552 },553 {554 "data": {555 "application/vnd.jupyter.widget-view+json": {556 "model_id": "0c561dab67914ea9b6e1aab803600551",557 "version_major": 2,558 "version_minor": 0559 },560 "text/plain": [561 "Map: 0%| | 0/227 [00:00<?, ? examples/s]"562 ]563 },564 "metadata": {},565 "output_type": "display_data"566 },567 {568 "data": {569 "text/plain": [570 "{'sentence': 'It will be operated by Nokia , and supported by its Nokia NetAct network and service management system .',\n",571 " 'label': 1,\n",572 " 'text_label': 'neutral'}"573 ]574 },575 "execution_count": 17,576 "metadata": {},577 "output_type": "execute_result"578 }579 ],580 "source": [581 "# loading dataset\n",582 "dataset = load_dataset(\"financial_phrasebank\", \"sentences_allagree\")\n",583 "dataset = dataset[\"train\"].train_test_split(test_size=0.1)\n",584 "dataset[\"validation\"] = dataset[\"test\"]\n",585 "del dataset[\"test\"]\n",586 "\n",587 "classes = dataset[\"train\"].features[\"label\"].names\n",588 "dataset = dataset.map(\n",589 " lambda x: {\"text_label\": [classes[label] for label in x[\"label\"]]},\n",590 " batched=True,\n",591 " num_proc=1,\n",592 ")\n",593 "\n",594 "dataset[\"train\"][0]"595 ]596 },597 {598 "cell_type": "code",599 "execution_count": 18,600 "metadata": {601 "colab": {602 "base_uri": "https://localhost:8080/",603 "height": 17,604 "referenced_widgets": [605 "e1e80a68a9e7429397cafc96c3c11f80",606 "5307864c2b1143f4b44f3f172611113e",607 "2e2b6c3f48974ea4aca9b7710a03379e",608 "aae78f9bd53348bda45967a38736cb78",609 "34db17e0f28d40d6abafb8acd5dda379",610 "8361dc2e0a834da6a0ad87f7b0cb4e1b",611 "56f1d9d56dd44c8aa923d09a59cb0ebc",612 "d93bfb366db14c2fa77b038752f69b38",613 "749aaa39135841f98b344ffb840df3d4",614 "5e5aa58adb0f48579871df33845e30b1",615 "c25b49b7adaa48a0a3a306aa1e0661b4",616 "21f582e1208a4a38ae3c0cdce87e5c14",617 "d9d37b8b79f24dbf837327a250a5a346",618 "8ba99043c350456d8623ce1d8c98f7a0",619 "8bf37c12d5f74f7d8dbba423a9ee3ac3",620 "f9d86ad7fa734f3a857505a542256a3c",621 "86bf02b06ed740a88015c2b944205c1e",622 "aef6a6be67f749908060d8038b6d3804",623 "664c02903cb248fb9339805bccfd6c1d",624 "82195b807b664a9585a76e0e50fe7609",625 "8621932be14f42858d841e2ac1b173e7",626 "71bcdb1e02144c9587879d8d815b91d4"627 ]628 },629 "id": "adf9608c",630 "outputId": "3e4bc95f-1dc4-4d34-c212-6d2374359673"631 },632 "outputs": [633 {634 "data": {635 "application/vnd.jupyter.widget-view+json": {636 "model_id": "e1e80a68a9e7429397cafc96c3c11f80",637 "version_major": 2,638 "version_minor": 0639 },640 "text/plain": [641 "Running tokenizer on dataset: 0%| | 0/2037 [00:00<?, ? examples/s]"642 ]643 },644 "metadata": {},645 "output_type": "display_data"646 },647 {648 "data": {649 "application/vnd.jupyter.widget-view+json": {650 "model_id": "21f582e1208a4a38ae3c0cdce87e5c14",651 "version_major": 2,652 "version_minor": 0653 },654 "text/plain": [655 "Running tokenizer on dataset: 0%| | 0/227 [00:00<?, ? examples/s]"656 ]657 },658 "metadata": {},659 "output_type": "display_data"660 }661 ],662 "source": [663 "# data preprocessing\n",664 "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",665 "\n",666 "\n",667 "def preprocess_function(examples):\n",668 " inputs = examples[text_column]\n",669 " targets = examples[label_column]\n",670 " model_inputs = tokenizer(inputs, max_length=max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",671 " labels = tokenizer(targets, max_length=3, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",672 " labels = labels[\"input_ids\"]\n",673 " labels[labels == tokenizer.pad_token_id] = -100\n",674 " model_inputs[\"labels\"] = labels\n",675 " return model_inputs\n",676 "\n",677 "\n",678 "processed_datasets = dataset.map(\n",679 " preprocess_function,\n",680 " batched=True,\n",681 " num_proc=1,\n",682 " remove_columns=dataset[\"train\"].column_names,\n",683 " load_from_cache_file=False,\n",684 " desc=\"Running tokenizer on dataset\",\n",685 ")\n",686 "\n",687 "train_dataset = processed_datasets[\"train\"]\n",688 "eval_dataset = processed_datasets[\"validation\"]\n",689 "\n",690 "train_dataloader = DataLoader(\n",691 " train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",692 ")\n",693 "eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"694 ]695 },696 {697 "cell_type": "code",698 "execution_count": 19,699 "metadata": {700 "id": "f733a3c6"701 },702 "outputs": [],703 "source": [704 "# optimizer and lr scheduler\n",705 "optimizer = torch.optim.AdamW(model.parameters(), lr=lr)\n",706 "lr_scheduler = get_linear_schedule_with_warmup(\n",707 " optimizer=optimizer,\n",708 " num_warmup_steps=0,\n",709 " num_training_steps=(len(train_dataloader) * num_epochs),\n",710 ")"711 ]712 },713 {714 "cell_type": "code",715 "execution_count": 20,716 "metadata": {717 "colab": {718 "base_uri": "https://localhost:8080/"719 },720 "id": "6b3a4090",721 "outputId": "369cfce9-90f2-47a1-8653-ea1168943949"722 },723 "outputs": [724 {725 "name": "stderr",726 "output_type": "stream",727 "text": [728 "100%|██████████| 255/255 [02:33<00:00, 1.67it/s]\n",729 "100%|██████████| 29/29 [00:08<00:00, 3.48it/s]\n"730 ]731 },732 {733 "name": "stdout",734 "output_type": "stream",735 "text": [736 "epoch=0: train_ppl=tensor(1.4939, device='cuda:0') train_epoch_loss=tensor(0.4014, device='cuda:0') eval_ppl=tensor(1.0514, device='cuda:0') eval_epoch_loss=tensor(0.0501, device='cuda:0')\n"737 ]738 },739 {740 "name": "stderr",741 "output_type": "stream",742 "text": [743 "100%|██████████| 255/255 [02:32<00:00, 1.67it/s]\n",744 "100%|██████████| 29/29 [00:08<00:00, 3.43it/s]\n"745 ]746 },747 {748 "name": "stdout",749 "output_type": "stream",750 "text": [751 "epoch=1: train_ppl=tensor(1.0523, device='cuda:0') train_epoch_loss=tensor(0.0510, device='cuda:0') eval_ppl=tensor(1.0383, device='cuda:0') eval_epoch_loss=tensor(0.0376, device='cuda:0')\n"752 ]753 },754 {755 "name": "stderr",756 "output_type": "stream",757 "text": [758 "100%|██████████| 255/255 [02:32<00:00, 1.68it/s]\n",759 "100%|██████████| 29/29 [00:08<00:00, 3.44it/s]"760 ]761 },762 {763 "name": "stdout",764 "output_type": "stream",765 "text": [766 "epoch=2: train_ppl=tensor(1.0397, device='cuda:0') train_epoch_loss=tensor(0.0389, device='cuda:0') eval_ppl=tensor(1.0392, device='cuda:0') eval_epoch_loss=tensor(0.0385, device='cuda:0')\n"767 ]768 },769 {770 "name": "stderr",771 "output_type": "stream",772 "text": [773 "\n"774 ]775 }776 ],777 "source": [778 "# training and evaluation\n",779 "model = model.to(device)\n",780 "\n",781 "for epoch in range(num_epochs):\n",782 " model.train()\n",783 " total_loss = 0\n",784 " for step, batch in enumerate(tqdm(train_dataloader)):\n",785 " batch = {k: v.to(device) for k, v in batch.items()}\n",786 " outputs = model(**batch)\n",787 " loss = outputs.loss\n",788 " total_loss += loss.detach().float()\n",789 " loss.backward()\n",790 " optimizer.step()\n",791 " lr_scheduler.step()\n",792 " optimizer.zero_grad()\n",793 "\n",794 " model.eval()\n",795 " eval_loss = 0\n",796 " eval_preds = []\n",797 " for step, batch in enumerate(tqdm(eval_dataloader)):\n",798 " batch = {k: v.to(device) for k, v in batch.items()}\n",799 " with torch.no_grad():\n",800 " outputs = model(**batch)\n",801 " loss = outputs.loss\n",802 " eval_loss += loss.detach().float()\n",803 " eval_preds.extend(\n",804 " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",805 " )\n",806 "\n",807 " eval_epoch_loss = eval_loss / len(eval_dataloader)\n",808 " eval_ppl = torch.exp(eval_epoch_loss)\n",809 " train_epoch_loss = total_loss / len(train_dataloader)\n",810 " train_ppl = torch.exp(train_epoch_loss)\n",811 " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"812 ]813 },814 {815 "cell_type": "code",816 "execution_count": 21,817 "metadata": {818 "colab": {819 "base_uri": "https://localhost:8080/"820 },821 "id": "6cafa67b",822 "outputId": "0db923d2-522c-4cb7-b694-6e2e20beae98"823 },824 "outputs": [825 {826 "name": "stdout",827 "output_type": "stream",828 "text": [829 "accuracy=96.91629955947137 % on the evaluation dataset\n",830 "eval_preds[:10]=['neutral', 'neutral', 'neutral', 'neutral', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n",831 "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'neutral', 'neutral', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n"832 ]833 }834 ],835 "source": [836 "# print accuracy\n",837 "correct = 0\n",838 "total = 0\n",839 "for pred, true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",840 " if pred.strip() == true.strip():\n",841 " correct += 1\n",842 " total += 1\n",843 "accuracy = correct / total * 100\n",844 "print(f\"{accuracy=} % on the evaluation dataset\")\n",845 "print(f\"{eval_preds[:10]=}\")\n",846 "print(f\"{dataset['validation']['text_label'][:10]=}\")"847 ]848 },849 {850 "cell_type": "code",851 "execution_count": 22,852 "metadata": {853 "id": "a8de6005"854 },855 "outputs": [],856 "source": [857 "# saving model\n",858 "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",859 "model.save_pretrained(peft_model_id)"860 ]861 },862 {863 "cell_type": "code",864 "execution_count": 23,865 "metadata": {866 "colab": {867 "base_uri": "https://localhost:8080/"868 },869 "id": "bd20cd4c",870 "outputId": "0f25d837-80b1-476f-c897-92c3fef04fb2"871 },872 "outputs": [873 {874 "name": "stdout",875 "output_type": "stream",876 "text": [877 "1.2M\tbigscience/mt0-large_IA3_SEQ_2_SEQ_LM/adapter_model.bin\n"878 ]879 }880 ],881 "source": [882 "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n",883 "!du -h $ckpt"884 ]885 },886 {887 "cell_type": "code",888 "execution_count": 24,889 "metadata": {890 "id": "76c2fc29"891 },892 "outputs": [],893 "source": [894 "from peft import PeftModel, PeftConfig\n",895 "\n",896 "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",897 "\n",898 "config = PeftConfig.from_pretrained(peft_model_id)\n",899 "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n",900 "model = PeftModel.from_pretrained(model, peft_model_id)"901 ]902 },903 {904 "cell_type": "code",905 "execution_count": 25,906 "metadata": {907 "colab": {908 "base_uri": "https://localhost:8080/"909 },910 "id": "37d712ce",911 "outputId": "4828819a-b640-4f6c-91e3-878b648e9a75"912 },913 "outputs": [914 {915 "name": "stdout",916 "output_type": "stream",917 "text": [918 "25 November 2010 - Finnish paints and coatings company Tikkurila Oyj ( HEL : TIK1V ) said today that Finnish state-owned investment company Solidium Oy sold its 14.7 % stake in the company for a total of EUR98m .\n",919 "{'input_ids': tensor([[ 877, 3277, 1068, 259, 264, 515, 143136, 42068, 263,\n",920 " 305, 259, 101264, 263, 5835, 22538, 4496, 2697, 20860,\n",921 " 385, 274, 76347, 259, 267, 259, 93686, 353, 561,\n",922 " 259, 271, 2426, 7883, 533, 515, 143136, 6509, 264,\n",923 " 45815, 37624, 5835, 35133, 16558, 20860, 22026, 2476, 5006,\n",924 " 487, 1448, 259, 96189, 281, 287, 5835, 332, 259,\n",925 " 262, 2725, 304, 2687, 5577, 282, 259, 260, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",926 " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",927 " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",928 "tensor([[ 0, 59006, 1]])\n",929 "['neutral']\n"930 ]931 }932 ],933 "source": [934 "model.eval()\n",935 "i = 13\n",936 "inputs = tokenizer(dataset[\"validation\"][text_column][i], return_tensors=\"pt\")\n",937 "print(dataset[\"validation\"][text_column][i])\n",938 "print(inputs)\n",939 "\n",940 "with torch.no_grad():\n",941 " outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n",942 " print(outputs)\n",943 " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"944 ]945 },946 {947 "cell_type": "code",948 "execution_count": null,949 "metadata": {950 "id": "66c65ea4"951 },952 "outputs": [],953 "source": []954 },955 {956 "cell_type": "code",957 "execution_count": null,958 "metadata": {959 "id": "65e71f78"960 },961 "outputs": [],962 "source": []963 }964 ],965 "metadata": {966 "accelerator": "GPU",967 "colab": {968 "gpuType": "T4",969 "machine_shape": "hm",970 "provenance": []971 },972 "kernelspec": {973 "display_name": "Python 3",974 "language": "python",975 "name": "python3"976 },977 "language_info": {978 "codemirror_mode": {979 "name": "ipython",980 "version": 3981 },982 "file_extension": ".py",983 "mimetype": "text/x-python",984 "name": "python",985 "nbconvert_exporter": "python",986 "pygments_lexer": "ipython3",987 "version": "3.8.3"988 },989 "vscode": {990 "interpreter": {991 "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"992 }993 },994 "widgets": {995 "application/vnd.jupyter.widget-state+json": {996 "013e3343285f437a893bdd673fb90e22": {997 "model_module": "@jupyter-widgets/controls",998 "model_module_version": "1.5.0",999 "model_name": "FloatProgressModel",1000 "state": {1001 "_dom_classes": [],1002 "_model_module": "@jupyter-widgets/controls",1003 "_model_module_version": "1.5.0",1004 "_model_name": "FloatProgressModel",1005 "_view_count": null,1006 "_view_module": "@jupyter-widgets/controls",1007 "_view_module_version": "1.5.0",1008 "_view_name": "ProgressView",1009 "bar_style": "",1010 "description": "",1011 "description_tooltip": null,1012 "layout": "IPY_MODEL_5b1bdaf16cbc473081e4237f839167b9",1013 "max": 227,1014 "min": 0,1015 "orientation": "horizontal",1016 "style": "IPY_MODEL_51f8fb45485540bb985b606d43ae04ea",1017 "value": 2271018 }1019 },1020 "0c561dab67914ea9b6e1aab803600551": {1021 "model_module": "@jupyter-widgets/controls",1022 "model_module_version": "1.5.0",1023 "model_name": "HBoxModel",1024 "state": {1025 "_dom_classes": [],1026 "_model_module": "@jupyter-widgets/controls",1027 "_model_module_version": "1.5.0",1028 "_model_name": "HBoxModel",1029 "_view_count": null,1030 "_view_module": "@jupyter-widgets/controls",1031 "_view_module_version": "1.5.0",1032 "_view_name": "HBoxView",1033 "box_style": "",1034 "children": [1035 "IPY_MODEL_1e021a1954b44d69a90101a96c360661",1036 "IPY_MODEL_013e3343285f437a893bdd673fb90e22",1037 "IPY_MODEL_28802da68fb04d70b1c6bc511a04676f"1038 ],1039 "layout": "IPY_MODEL_94174da0d6554be087d4527bea5b511a"1040 }1041 },1042 "1b8aada826a0451bb60c418b19178c8c": {1043 "model_module": "@jupyter-widgets/base",1044 "model_module_version": "1.2.0",1045 "model_name": "LayoutModel",1046 "state": {1047 "_model_module": "@jupyter-widgets/base",1048 "_model_module_version": "1.2.0",1049 "_model_name": "LayoutModel",1050 "_view_count": null,1051 "_view_module": "@jupyter-widgets/base",1052 "_view_module_version": "1.2.0",1053 "_view_name": "LayoutView",1054 "align_content": null,1055 "align_items": null,1056 "align_self": null,1057 "border": null,1058 "bottom": null,1059 "display": null,1060 "flex": null,1061 "flex_flow": null,1062 "grid_area": null,1063 "grid_auto_columns": null,1064 "grid_auto_flow": null,1065 "grid_auto_rows": null,1066 "grid_column": null,1067 "grid_gap": null,1068 "grid_row": null,1069 "grid_template_areas": null,1070 "grid_template_columns": null,1071 "grid_template_rows": null,1072 "height": null,1073 "justify_content": null,1074 "justify_items": null,1075 "left": null,1076 "margin": null,1077 "max_height": null,1078 "max_width": null,1079 "min_height": null,1080 "min_width": null,1081 "object_fit": null,1082 "object_position": null,1083 "order": null,1084 "overflow": null,1085 "overflow_x": null,1086 "overflow_y": null,1087 "padding": null,1088 "right": null,1089 "top": null,1090 "visibility": null,1091 "width": null1092 }1093 },1094 "1bcba805972b484d8b6aa6542c81841c": {1095 "model_module": "@jupyter-widgets/base",1096 "model_module_version": "1.2.0",1097 "model_name": "LayoutModel",1098 "state": {1099 "_model_module": "@jupyter-widgets/base",1100 "_model_module_version": "1.2.0",1101 "_model_name": "LayoutModel",1102 "_view_count": null,1103 "_view_module": "@jupyter-widgets/base",1104 "_view_module_version": "1.2.0",1105 "_view_name": "LayoutView",1106 "align_content": null,1107 "align_items": null,1108 "align_self": null,1109 "border": null,1110 "bottom": null,1111 "display": null,1112 "flex": null,1113 "flex_flow": null,1114 "grid_area": null,1115 "grid_auto_columns": null,1116 "grid_auto_flow": null,1117 "grid_auto_rows": null,1118 "grid_column": null,1119 "grid_gap": null,1120 "grid_row": null,1121 "grid_template_areas": null,1122 "grid_template_columns": null,1123 "grid_template_rows": null,1124 "height": null,1125 "justify_content": null,1126 "justify_items": null,1127 "left": null,1128 "margin": null,1129 "max_height": null,1130 "max_width": null,1131 "min_height": null,1132 "min_width": null,1133 "object_fit": null,1134 "object_position": null,1135 "order": null,1136 "overflow": null,1137 "overflow_x": null,1138 "overflow_y": null,1139 "padding": null,1140 "right": null,1141 "top": null,1142 "visibility": null,1143 "width": null1144 }1145 },1146 "1e021a1954b44d69a90101a96c360661": {1147 "model_module": "@jupyter-widgets/controls",1148 "model_module_version": "1.5.0",1149 "model_name": "HTMLModel",1150 "state": {1151 "_dom_classes": [],1152 "_model_module": "@jupyter-widgets/controls",1153 "_model_module_version": "1.5.0",1154 "_model_name": "HTMLModel",1155 "_view_count": null,1156 "_view_module": "@jupyter-widgets/controls",1157 "_view_module_version": "1.5.0",1158 "_view_name": "HTMLView",1159 "description": "",1160 "description_tooltip": null,1161 "layout": "IPY_MODEL_dc8ab16a1e6c4e6893c95ccd16568f9a",1162 "placeholder": "",1163 "style": "IPY_MODEL_72383136663448d89cf3b82b87cbb392",1164 "value": "Map: 0%"1165 }1166 },1167 "21f582e1208a4a38ae3c0cdce87e5c14": {1168 "model_module": "@jupyter-widgets/controls",1169 "model_module_version": "1.5.0",1170 "model_name": "HBoxModel",1171 "state": {1172 "_dom_classes": [],1173 "_model_module": "@jupyter-widgets/controls",1174 "_model_module_version": "1.5.0",1175 "_model_name": "HBoxModel",1176 "_view_count": null,1177 "_view_module": "@jupyter-widgets/controls",1178 "_view_module_version": "1.5.0",1179 "_view_name": "HBoxView",1180 "box_style": "",1181 "children": [1182 "IPY_MODEL_d9d37b8b79f24dbf837327a250a5a346",1183 "IPY_MODEL_8ba99043c350456d8623ce1d8c98f7a0",1184 "IPY_MODEL_8bf37c12d5f74f7d8dbba423a9ee3ac3"1185 ],1186 "layout": "IPY_MODEL_f9d86ad7fa734f3a857505a542256a3c"1187 }1188 },1189 "28802da68fb04d70b1c6bc511a04676f": {1190 "model_module": "@jupyter-widgets/controls",1191 "model_module_version": "1.5.0",1192 "model_name": "HTMLModel",1193 "state": {1194 "_dom_classes": [],1195 "_model_module": "@jupyter-widgets/controls",1196 "_model_module_version": "1.5.0",1197 "_model_name": "HTMLModel",1198 "_view_count": null,1199 "_view_module": "@jupyter-widgets/controls",1200 "_view_module_version": "1.5.0",