PEFT/sequence-classification
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 2,6 "id": "a9935ae2",7 "metadata": {8 "id": "a9935ae2"9 },10 "outputs": [],11 "source": [12 "import argparse\n",13 "import os\n",14 "\n",15 "import torch\n",16 "from torch.optim import AdamW\n",17 "from torch.utils.data import DataLoader\n",18 "import peft\n",19 "\n",20 "import evaluate\n",21 "from datasets import load_dataset\n",22 "from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",23 "from tqdm import tqdm"24 ]25 },26 {27 "cell_type": "code",28 "execution_count": 3,29 "id": "e3b13308",30 "metadata": {31 "id": "e3b13308"32 },33 "outputs": [],34 "source": [35 "batch_size = 8\n",36 "model_name_or_path = \"roberta-large\"\n",37 "task = \"mrpc\"\n",38 "peft_type = peft.PeftType.IA3\n",39 "device = \"cuda\"\n",40 "num_epochs = 12"41 ]42 },43 {44 "cell_type": "code",45 "execution_count": 4,46 "id": "0526f571",47 "metadata": {48 "id": "0526f571"49 },50 "outputs": [],51 "source": [52 "# peft_config = LoraConfig(task_type=\"SEQ_CLS\", inference_mode=False, r=8, lora_alpha=16, lora_dropout=0.1)\n",53 "peft_config = peft.IA3Config(task_type=\"SEQ_CLS\", inference_mode=False)\n",54 "lr = 1e-3"55 ]56 },57 {58 "cell_type": "code",59 "execution_count": 5,60 "id": "c2697d07",61 "metadata": {62 "colab": {63 "base_uri": "https://localhost:8080/",64 "height": 489,65 "referenced_widgets": [66 "6ea6ff70fa164264aef9efce9f921f10",67 "ecc2102ede2d4c8b94ce66b247054c96",68 "e64ca48867434ca2944dcb2b1c70c02c",69 "7ac75048225f4a4bbedac97965cf9837",70 "332971de5e894c8ca866c311cc6e180c",71 "bdf0999dfddc43ca8e04ceaac628064c",72 "c289713b179641bd915b3c334208197a",73 "e3d36dd4ddcb4287b8d8c942a26dc478",74 "0b20fa2eb59749e1b564df70f2378984",75 "d8e645f1697d46e0ac23f70e14498fab",76 "8dc1453856d549c4a1a688818447dd59",77 "d267c0cf26a8466d9938861ff5272a1e",78 "561f61d1cec94f5abbff3f8746c3fd96",79 "a5c93fac02884914ae8e140e0c0d2a17",80 "96c4ecd7376c43ee9a4ba270851d6fff",81 "5062d6f6feb34835afaf6d452900d514",82 "a05844fa4675494cb7bbe48f40f1aaac",83 "b241da7afea94fc0a6b407a0a78a8355",84 "74e056313e9e4a92bfaa22019ac1e58e",85 "938a5b44d90140e29ba33628a2215f2a",86 "1f27766bc2d941f6a1d03fc69a6026a3",87 "1c5e43a201f0460f88b53739bc1eaa43",88 "24e236a4360e416a8e5c20d887274bcc",89 "7011752f7e0a422883bf0f218f6941c3",90 "248fe45eee37449982520a890696e6d4",91 "5ec5d0d9191047608f61cb78563f7641",92 "d7c2b00fd90147528b00a29552f29b44",93 "ea307bb7a9ad46a484330b1daf708169",94 "27009ed667c242488773ce3fcc58360a",95 "faf0c550adc5402bbee71b027822ce9c",96 "9c57da41dfc04fe4a4437097285599bf",97 "d4453448f8b04f0ea76c71887bd33a8c",98 "ff5172169c794d40b6c433da642d54a9",99 "788a143aad46467789acf08762fbf39f",100 "32ba35144ab34b0ebb7cfb75b86ccdd4",101 "3ef39b223ab74d788223970ec0da21e6",102 "be40292e613949a0ba1bfd1846cdab92",103 "a6ab0e37d063407fa4adc11f2f0299da",104 "71c3a5a515a949bfb2290612bfb2d05f",105 "1232101b618d43f5889e4ad81fa24514",106 "2204ca891c3940879abde0f55fbadf03",107 "58f2da1b793b44b09cedb5e0a3a1ef02",108 "51ec9bdff1834e4f8390f0eaf7d4aeb8",109 "71b4a798dc374a72817f6c118f2f05b8",110 "a439e5ecb0a040ca8188c6d74ff643a9",111 "710f512e4a2b40cb85805b33a7dc42e4",112 "e5994b94b84143f4a06c49a45a078bd0",113 "e4f935e4e8e84320b2381aaf3493962c",114 "44eb5aeeb0004f799e79d3ed51cf37d4",115 "0c488289ee494a8d99d1f02a13729382",116 "49ccb14ddd874e798371494942725128",117 "015132b18da54a8f89a83cd9bf6fd17c",118 "39f26160766f48798761079527e14396",119 "5408dfdfc5624ff5b1c0b2e494dd4b35",120 "9c50277ed18a424a9da98707cf726d29",121 "11ba7f922c58474d9cb8b8c7a22caddc",122 "598fc42dfed04aadaeb38539dd259871",123 "fb05d1ece2ba4963bcbf99f89296c709",124 "e3fe73a6ffcf4d089911b4149b9b4512",125 "45e1b59770f946dba06511f9b5d1ad23",126 "2c512bc0a21a4f6684aa0593a969e6cb",127 "376f00f38b46434e922c3f6f7dc4a85c",128 "e815b0749289411eb680c56e4fd39dae",129 "33e9821903f84551a2e56c0586a5332b",130 "0b2e59615f80452cbe25dfd467099b84",131 "45128b02652b4c20bf448db8495d76d2",132 "d1997b08e5284008afce56a5ed6347be",133 "730111ee99b4470c81ab4ade07b30352",134 "562a55ffabef4d218c78c5dcc6665484",135 "59e1d01621f4404ca2102d00e2b01f87",136 "1896a8f160ea45e5ac2a88970feecdb9",137 "7d6d21774fec4eb0bab3bc2bfa2708d6",138 "9882a67915f8475c9f4b8a5b15d64e50",139 "47ffc35023144c4fa6b93e73b7a2ee60",140 "16f8d52980dc41e89ed3bfb9172420a6",141 "2bc5432ab9464f7b9192aa1f5a26a1b2",142 "80bcbdb92af34c9889b7d105affe34ba",143 "6786c3895b594301b2a7d4ed2767cb35",144 "bf35a5a5d9b840a18ef10938d23fce0c",145 "ad4872e198ee44f8b83323d891347ffa",146 "0f29f96aec7e44a1b97cda0fcbe17665",147 "dbb4032fab4d49a5b250be57a4d51fb0",148 "af185d9bedf64d54bc77f7f6e7c448f4",149 "4d9d717d2226444094e9753fdc843849",150 "3d323bc6e1fd4091a0ed2ffd521f2ec7",151 "2616b6102108476f8bf9e3d35b63494d",152 "844ea1050893482785c1b68150f4ab20",153 "92d023972a204222b59c12fc4b4d3bcf",154 "7900796766b946da886338653b495533",155 "84b50ae864e0463d98efa792e149e712",156 "802dc985a31e4febb8eafa4682e242ec",157 "f31d9d6e2d3140eeb821153a7b69b90a",158 "3f2d9eba845341da97fa1b957e72de5e",159 "d3638a2985fc4fdfbf239914dbc32fe8",160 "a9d237f2f62e4a839abdec541715a5de",161 "b451310e94b74abda4e795a59cdda9ab",162 "537c7d2f4260498a82132cb9fae5daa5",163 "6f9efa5f778d4f029dca7b4d6817b4c4",164 "d29a5f422214434c9be3886ce0d1e918",165 "e0e903b3ae8044f4a58ecd70825f36ec",166 "99e8679e71f4443c92f13971e8885d38",167 "55a0ca0754c94f0bba6ede50b7fb7ea8",168 "fae83f7a625448e788ffdb1a13d2d530",169 "3086db94647a41569b6f091e4f11f3cb",170 "24062742dc6749a2aab19cbfd1e11684",171 "1b6c8de76fd948b5b0eba8346f6e0bec",172 "276fc5825a624a3a9026301620682c6c",173 "5c10a16929ce40e48361efa44e96a7f8",174 "53f9d507f4024b2080591215d3c4bab4",175 "c77cdd4479294ce7bf3f3ef271e95b1d",176 "e66a0c2552ff46b5a41cf8d803114b8f",177 "897708ca40da48f3ae846710b4e7f7f0",178 "1aa34d9a0b16444baadf2ed7e43b72e5",179 "01f39b548eff49aea6ba0efabb4486de",180 "46fc201246804f28946fd0c59e09c4ac",181 "e6c085b451a346b6a3b82c67d7b9e9e2",182 "29a8cce77d2745038f2c742d17254139",183 "326b48d7d2e94defa52858171177e7d7",184 "9a3dcc1c2fe542b7b912419935d9dd9a",185 "b82b0c29bd53432baa5596b4f1aaf076",186 "a8ccd811bba848d2ad2f84d2cdf9ef03",187 "9ae4e6a6543e4da6bba06175aeef3ea1",188 "fa9675039cc443f791dcfbc0bdca065c",189 "5b0a0578afad40868bb71707c05e0335",190 "d1a82e42e866449cbb70c1f68ac1bc03",191 "bd12886eee644913ba83fd4ebb6b62ee",192 "fc11a9d0c0b7409cb2f61fcabac2bfa6",193 "9cf73d46c2f04bf398a4564a56f03bd6",194 "f46d3699775b4d7193adeebb4f18a34f",195 "5997f14e0a3044249b9cabc2b307e3a4",196 "5ba9f4bd64bd4e1e905887760f90ae3e",197 "1a8e75c718d14f4c8f99de1c8efa84b5",198 "786b2430a42942f1aeae253861820dcb",199 "e2741973e91745ea930d3cc23070cf52",200 "b9f18367c54b4203bee70c680ae9cdde",201 "7db82bb6fa2949b3a6bc9eb152fc3af1",202 "136a32dbeef646df944a8b59cb00c0c4",203 "c2fa0ef9f45b447e9dfe5c576428c714",204 "241dd21eeb5843ef8433e47b415c5b62",205 "c9837f88650844ea94442ad1c5682972",206 "963da028a1594c1bbd223e93832f44bd",207 "d981db72ab2745a598ed45a8762d5fcd",208 "73379eb4954d4be6ab90008addd7d3bd",209 "007f2f512694405c9245edd5e1f58551",210 "9aaeeb1213854768a6af51f8db54f6b6",211 "e6e2192c0a904bb6850c6bc68b579995",212 "608a68c6f651419d8fd210044b4561cf",213 "ac9d6255521f428fa7fda5735d71dbec",214 "a39ea243b7964c298eae71e9eaf32e17",215 "bf7f7cb363df4384ade192666b77c715",216 "841f3571fcec42ad8e1e43997567788c",217 "a246e6c60c054b4383c510c31b255a87",218 "da3e335ddc9846ea832c75d69684575f",219 "daa8ce62f0204a11bde7cd9e31cc2fa1",220 "6b937f0053b64c67af74d44bcb6e51b7",221 "72f55658a145483cb90668bf7b6f6c8a",222 "357c082907e140b58c807a394446d811",223 "f127594b694f4d2fafb3872e8190b1d6",224 "f9c8bd12201a4c8a86b2bb3c1a14d74e",225 "dc55ec8e57984efcb0f269ef0ef41c02",226 "cd07261ef21a4b859de53f244df33f2a",227 "c47de12dd692434ab9497e8a0d2a19ba",228 "963b22b2ef1e45f5ba8320f72ea6a83b",229 "3e2ca33b7473499d8314238ec245cd90",230 "e3313a7fb356432bada289ad346f1bdb",231 "75ee099566ca46eb8a75475246aaab01",232 "e94dd1ea928f43b880af21ced7f11d14",233 "ad84cdcfa07f4f608dccd10395d35e79",234 "8bd3a235ae654b1cbaae36aeb1a62e70",235 "ebe2f83a5783401ab501cdaf9c4e2ad5",236 "fe20644617844cf7a471e22a99ca7b5e",237 "d5502b7dcf784d289fc748881924334c",238 "6bf5b27d3c3c407eb10f8d9d6e8e8d22",239 "47480982ce83440997297e605cdf8a31",240 "32764784cc9644198c622f6706db6836",241 "83bae5949ec448c6ad6f68a7b8d3d436",242 "a018f7c156584eb3833b6aba3710b3c0",243 "c2a078c69d3f45ca9895e0e7f95aaf2b",244 "8c55316c54d44bd4b75a8457e2d8c595",245 "1824b7b748ee4763b5098cd915107f63",246 "87b93c14cdae477ab49522354631e82b",247 "a8b5d642ed654c0f85f3a9610c68b754",248 "8750ab964b5444d49db4fe8542964d8c",249 "c994a28df165445bbf0d80c165bc5a0b",250 "1a2897c84d454c1a9c115aef178f4fcf",251 "b668d56ab549478984ad14c22e040e47",252 "bac855f4c7044fe88bcd74170e13f103",253 "f871eeaa54cf451b8a2de64eed90d5b6",254 "39492d8b91f64e97bc8caead77957508",255 "1487d054f1824f739c93c00621591bd0",256 "cca43d628ff94369bc9713cbad616adb",257 "6aa4308f0cb348419f1217e48ef2dbd7",258 "d9db044ff31f4856a4d25e57fe9882bc",259 "f76b651681d74b2eab0ab07f43983a2a",260 "1e3adfe9c9e34d4abe74b65e298c0e7b",261 "f91ed6a931c247b5903304a28998633d",262 "34e576af7aab4ef795239b1d9a281b15",263 "9d8e3a4fe5864a4ca229ba6e092181e4",264 "e41888b1dea140d18be25efc6d99d0c3",265 "d63577f0ae724938952f9b681a89512a",266 "83623082d4e9457a9173db3129154f94",267 "4c1798c3d0cb43949a1ed7519b57d3fc",268 "6513f2b58d654a488b132abd8c83c9b6",269 "c4aa7cee6e5d4a89a2c33b9ce2e9c489",270 "b42e835e0ba24a08b5815234a22b4da6",271 "e8a53956f6ad46d58ec3a9a039f2303e",272 "91f07c6089354ceeb4e628ef1791d2d6",273 "c137c84ae40049ffba7d1f4d77b21de2",274 "81f8db0dcdbc4e5593fa3fbb6c2b9361"275 ]276 },277 "id": "c2697d07",278 "outputId": "c8318b3d-b6a0-4f0d-9903-54beb2baac75"279 },280 "outputs": [281 {282 "output_type": "display_data",283 "data": {284 "text/plain": [285 "Downloading (…)lve/main/config.json: 0%| | 0.00/482 [00:00<?, ?B/s]"286 ],287 "application/vnd.jupyter.widget-view+json": {288 "version_major": 2,289 "version_minor": 0,290 "model_id": "6ea6ff70fa164264aef9efce9f921f10"291 }292 },293 "metadata": {}294 },295 {296 "output_type": "display_data",297 "data": {298 "text/plain": [299 "Downloading (…)olve/main/vocab.json: 0%| | 0.00/899k [00:00<?, ?B/s]"300 ],301 "application/vnd.jupyter.widget-view+json": {302 "version_major": 2,303 "version_minor": 0,304 "model_id": "d267c0cf26a8466d9938861ff5272a1e"305 }306 },307 "metadata": {}308 },309 {310 "output_type": "display_data",311 "data": {312 "text/plain": [313 "Downloading (…)olve/main/merges.txt: 0%| | 0.00/456k [00:00<?, ?B/s]"314 ],315 "application/vnd.jupyter.widget-view+json": {316 "version_major": 2,317 "version_minor": 0,318 "model_id": "24e236a4360e416a8e5c20d887274bcc"319 }320 },321 "metadata": {}322 },323 {324 "output_type": "display_data",325 "data": {326 "text/plain": [327 "Downloading (…)/main/tokenizer.json: 0%| | 0.00/1.36M [00:00<?, ?B/s]"328 ],329 "application/vnd.jupyter.widget-view+json": {330 "version_major": 2,331 "version_minor": 0,332 "model_id": "788a143aad46467789acf08762fbf39f"333 }334 },335 "metadata": {}336 },337 {338 "output_type": "display_data",339 "data": {340 "text/plain": [341 "Downloading builder script: 0%| | 0.00/28.8k [00:00<?, ?B/s]"342 ],343 "application/vnd.jupyter.widget-view+json": {344 "version_major": 2,345 "version_minor": 0,346 "model_id": "a439e5ecb0a040ca8188c6d74ff643a9"347 }348 },349 "metadata": {}350 },351 {352 "output_type": "display_data",353 "data": {354 "text/plain": [355 "Downloading metadata: 0%| | 0.00/28.7k [00:00<?, ?B/s]"356 ],357 "application/vnd.jupyter.widget-view+json": {358 "version_major": 2,359 "version_minor": 0,360 "model_id": "11ba7f922c58474d9cb8b8c7a22caddc"361 }362 },363 "metadata": {}364 },365 {366 "output_type": "display_data",367 "data": {368 "text/plain": [369 "Downloading readme: 0%| | 0.00/27.9k [00:00<?, ?B/s]"370 ],371 "application/vnd.jupyter.widget-view+json": {372 "version_major": 2,373 "version_minor": 0,374 "model_id": "d1997b08e5284008afce56a5ed6347be"375 }376 },377 "metadata": {}378 },379 {380 "output_type": "stream",381 "name": "stdout",382 "text": [383 "Downloading and preparing dataset glue/mrpc to /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad...\n"384 ]385 },386 {387 "output_type": "display_data",388 "data": {389 "text/plain": [390 "Downloading data files: 0%| | 0/3 [00:00<?, ?it/s]"391 ],392 "application/vnd.jupyter.widget-view+json": {393 "version_major": 2,394 "version_minor": 0,395 "model_id": "6786c3895b594301b2a7d4ed2767cb35"396 }397 },398 "metadata": {}399 },400 {401 "output_type": "display_data",402 "data": {403 "text/plain": [404 "Downloading data: 0.00B [00:00, ?B/s]"405 ],406 "application/vnd.jupyter.widget-view+json": {407 "version_major": 2,408 "version_minor": 0,409 "model_id": "7900796766b946da886338653b495533"410 }411 },412 "metadata": {}413 },414 {415 "output_type": "display_data",416 "data": {417 "text/plain": [418 "Downloading data: 0.00B [00:00, ?B/s]"419 ],420 "application/vnd.jupyter.widget-view+json": {421 "version_major": 2,422 "version_minor": 0,423 "model_id": "e0e903b3ae8044f4a58ecd70825f36ec"424 }425 },426 "metadata": {}427 },428 {429 "output_type": "display_data",430 "data": {431 "text/plain": [432 "Downloading data: 0.00B [00:00, ?B/s]"433 ],434 "application/vnd.jupyter.widget-view+json": {435 "version_major": 2,436 "version_minor": 0,437 "model_id": "e66a0c2552ff46b5a41cf8d803114b8f"438 }439 },440 "metadata": {}441 },442 {443 "output_type": "display_data",444 "data": {445 "text/plain": [446 "Generating train split: 0%| | 0/3668 [00:00<?, ? examples/s]"447 ],448 "application/vnd.jupyter.widget-view+json": {449 "version_major": 2,450 "version_minor": 0,451 "model_id": "9ae4e6a6543e4da6bba06175aeef3ea1"452 }453 },454 "metadata": {}455 },456 {457 "output_type": "display_data",458 "data": {459 "text/plain": [460 "Generating validation split: 0%| | 0/408 [00:00<?, ? examples/s]"461 ],462 "application/vnd.jupyter.widget-view+json": {463 "version_major": 2,464 "version_minor": 0,465 "model_id": "786b2430a42942f1aeae253861820dcb"466 }467 },468 "metadata": {}469 },470 {471 "output_type": "display_data",472 "data": {473 "text/plain": [474 "Generating test split: 0%| | 0/1725 [00:00<?, ? examples/s]"475 ],476 "application/vnd.jupyter.widget-view+json": {477 "version_major": 2,478 "version_minor": 0,479 "model_id": "007f2f512694405c9245edd5e1f58551"480 }481 },482 "metadata": {}483 },484 {485 "output_type": "stream",486 "name": "stdout",487 "text": [488 "Dataset glue downloaded and prepared to /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad. Subsequent calls will reuse this data.\n"489 ]490 },491 {492 "output_type": "display_data",493 "data": {494 "text/plain": [495 " 0%| | 0/3 [00:00<?, ?it/s]"496 ],497 "application/vnd.jupyter.widget-view+json": {498 "version_major": 2,499 "version_minor": 0,500 "model_id": "6b937f0053b64c67af74d44bcb6e51b7"501 }502 },503 "metadata": {}504 },505 {506 "output_type": "display_data",507 "data": {508 "text/plain": [509 "Downloading builder script: 0%| | 0.00/5.75k [00:00<?, ?B/s]"510 ],511 "application/vnd.jupyter.widget-view+json": {512 "version_major": 2,513 "version_minor": 0,514 "model_id": "75ee099566ca46eb8a75475246aaab01"515 }516 },517 "metadata": {}518 },519 {520 "output_type": "display_data",521 "data": {522 "text/plain": [523 "Map: 0%| | 0/3668 [00:00<?, ? examples/s]"524 ],525 "application/vnd.jupyter.widget-view+json": {526 "version_major": 2,527 "version_minor": 0,528 "model_id": "a018f7c156584eb3833b6aba3710b3c0"529 }530 },531 "metadata": {}532 },533 {534 "output_type": "display_data",535 "data": {536 "text/plain": [537 "Map: 0%| | 0/408 [00:00<?, ? examples/s]"538 ],539 "application/vnd.jupyter.widget-view+json": {540 "version_major": 2,541 "version_minor": 0,542 "model_id": "f871eeaa54cf451b8a2de64eed90d5b6"543 }544 },545 "metadata": {}546 },547 {548 "output_type": "display_data",549 "data": {550 "text/plain": [551 "Map: 0%| | 0/1725 [00:00<?, ? examples/s]"552 ],553 "application/vnd.jupyter.widget-view+json": {554 "version_major": 2,555 "version_minor": 0,556 "model_id": "e41888b1dea140d18be25efc6d99d0c3"557 }558 },559 "metadata": {}560 }561 ],562 "source": [563 "if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",564 " padding_side = \"left\"\n",565 "else:\n",566 " padding_side = \"right\"\n",567 "\n",568 "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",569 "if getattr(tokenizer, \"pad_token_id\") is None:\n",570 " tokenizer.pad_token_id = tokenizer.eos_token_id\n",571 "\n",572 "datasets = load_dataset(\"glue\", task)\n",573 "metric = evaluate.load(\"glue\", task)\n",574 "\n",575 "\n",576 "def tokenize_function(examples):\n",577 " # max_length=None => use the model max length (it's actually the default)\n",578 " outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",579 " return outputs\n",580 "\n",581 "\n",582 "tokenized_datasets = datasets.map(\n",583 " tokenize_function,\n",584 " batched=True,\n",585 " remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",586 ")\n",587 "\n",588 "# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",589 "# transformers library\n",590 "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",591 "\n",592 "\n",593 "def collate_fn(examples):\n",594 " return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",595 "\n",596 "\n",597 "# Instantiate dataloaders.\n",598 "train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",599 "eval_dataloader = DataLoader(\n",600 " tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",601 ")\n",602 "test_dataloader = DataLoader(tokenized_datasets[\"test\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size)"603 ]604 },605 {606 "cell_type": "code",607 "execution_count": 6,608 "id": "2ed5ac74",609 "metadata": {610 "colab": {611 "base_uri": "https://localhost:8080/",612 "height": 1000,613 "referenced_widgets": [614 "0cecb897c86c4892b94a1990ab08a926",615 "b8af0294819e4280ad41fa1c11006adf",616 "c7530d63b2f745e799713284abacbd2c",617 "12a1e302a69543c5bc0e0a66be008ca0",618 "9c372e9e9b20433faed8530ca0f4424c",619 "b07f26a21325493cac19113f1aa1ee96",620 "fbdf6c544fb54294903524a69384e773",621 "6c6f2223243b4a7485aef7fbbfe07668",622 "a93509d61ac94628a74bc0f98c0eec06",623 "3a8de0eb7db44647a734590b6b351b44",624 "64d8affd2e854a1c9043fec7ca8a2796"625 ]626 },627 "id": "2ed5ac74",628 "outputId": "18ea15ac-ed8d-4d80-b166-706681ee49ab"629 },630 "outputs": [631 {632 "output_type": "display_data",633 "data": {634 "text/plain": [635 "Downloading model.safetensors: 0%| | 0.00/1.42G [00:00<?, ?B/s]"636 ],637 "application/vnd.jupyter.widget-view+json": {638 "version_major": 2,639 "version_minor": 0,640 "model_id": "0cecb897c86c4892b94a1990ab08a926"641 }642 },643 "metadata": {}644 },645 {646 "output_type": "stream",647 "name": "stderr",648 "text": [649 "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias']\n",650 "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",651 "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",652 "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",653 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"654 ]655 },656 {657 "output_type": "stream",658 "name": "stdout",659 "text": [660 "trainable params: 2,275,332 || all params: 356,585,476 || trainable%: 0.6380888042675075\n"661 ]662 },663 {664 "output_type": "execute_result",665 "data": {666 "text/plain": [667 "PeftModelForSequenceClassification(\n",668 " (base_model): IA3Model(\n",669 " (model): RobertaForSequenceClassification(\n",670 " (roberta): RobertaModel(\n",671 " (embeddings): RobertaEmbeddings(\n",672 " (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",673 " (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",674 " (token_type_embeddings): Embedding(1, 1024)\n",675 " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",676 " (dropout): Dropout(p=0.1, inplace=False)\n",677 " )\n",678 " (encoder): RobertaEncoder(\n",679 " (layer): ModuleList(\n",680 " (0-23): 24 x RobertaLayer(\n",681 " (attention): RobertaAttention(\n",682 " (self): RobertaSelfAttention(\n",683 " (query): Linear(in_features=1024, out_features=1024, bias=True)\n",684 " (key): Linear(\n",685 " in_features=1024, out_features=1024, bias=True\n",686 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",687 " )\n",688 " (value): Linear(\n",689 " in_features=1024, out_features=1024, bias=True\n",690 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1024x1])\n",691 " )\n",692 " (dropout): Dropout(p=0.1, inplace=False)\n",693 " )\n",694 " (output): RobertaSelfOutput(\n",695 " (dense): Linear(\n",696 " in_features=1024, out_features=1024, bias=True\n",697 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1x1024])\n",698 " )\n",699 " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",700 " (dropout): Dropout(p=0.1, inplace=False)\n",701 " )\n",702 " )\n",703 " (intermediate): RobertaIntermediate(\n",704 " (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",705 " (intermediate_act_fn): GELUActivation()\n",706 " )\n",707 " (output): RobertaOutput(\n",708 " (dense): Linear(\n",709 " in_features=4096, out_features=1024, bias=True\n",710 " (ia3_l): ParameterDict( (default): Parameter containing: [torch.FloatTensor of size 1x4096])\n",711 " )\n",712 " (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",713 " (dropout): Dropout(p=0.1, inplace=False)\n",714 " )\n",715 " )\n",716 " )\n",717 " )\n",718 " )\n",719 " (classifier): ModulesToSaveWrapper(\n",720 " (original_module): RobertaClassificationHead(\n",721 " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",722 " (dropout): Dropout(p=0.1, inplace=False)\n",723 " (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",724 " )\n",725 " (modules_to_save): ModuleDict(\n",726 " (default): RobertaClassificationHead(\n",727 " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",728 " (dropout): Dropout(p=0.1, inplace=False)\n",729 " (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",730 " )\n",731 " )\n",732 " )\n",733 " )\n",734 " )\n",735 ")"736 ]737 },738 "metadata": {},739 "execution_count": 6740 }741 ],742 "source": [743 "model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",744 "model = peft.get_peft_model(model, peft_config)\n",745 "model.print_trainable_parameters()\n",746 "model"747 ]748 },749 {750 "cell_type": "code",751 "execution_count": 7,752 "id": "0d2d0381",753 "metadata": {754 "id": "0d2d0381"755 },756 "outputs": [],757 "source": [758 "optimizer = AdamW(params=model.parameters(), lr=lr)\n",759 "\n",760 "# Instantiate scheduler\n",761 "lr_scheduler = get_linear_schedule_with_warmup(\n",762 " optimizer=optimizer,\n",763 " num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",764 " num_training_steps=(len(train_dataloader) * num_epochs),\n",765 ")"766 ]767 },768 {769 "cell_type": "code",770 "execution_count": 8,771 "id": "fa0e73be",772 "metadata": {773 "colab": {774 "base_uri": "https://localhost:8080/"775 },776 "id": "fa0e73be",777 "outputId": "bb17c146-8acc-477d-8f9f-65b8be794abb"778 },779 "outputs": [780 {781 "output_type": "stream",782 "name": "stderr",783 "text": [784 " 0%| | 0/459 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",785 "100%|██████████| 459/459 [01:41<00:00, 4.52it/s]\n",786 "100%|██████████| 51/51 [00:05<00:00, 8.89it/s]\n"787 ]788 },789 {790 "output_type": "stream",791 "name": "stdout",792 "text": [793 "epoch 0: {'accuracy': 0.7034313725490197, 'f1': 0.8212703101920238}\n"794 ]795 },796 {797 "output_type": "stream",798 "name": "stderr",799 "text": [800 "100%|██████████| 459/459 [01:45<00:00, 4.35it/s]\n",801 "100%|██████████| 51/51 [00:05<00:00, 8.66it/s]\n"802 ]803 },804 {805 "output_type": "stream",806 "name": "stdout",807 "text": [808 "epoch 1: {'accuracy': 0.7794117647058824, 'f1': 0.8432055749128919}\n"809 ]810 },811 {812 "output_type": "stream",813 "name": "stderr",814 "text": [815 "100%|██████████| 459/459 [01:47<00:00, 4.26it/s]\n",816 "100%|██████████| 51/51 [00:05<00:00, 8.50it/s]\n"817 ]818 },819 {820 "output_type": "stream",821 "name": "stdout",822 "text": [823 "epoch 2: {'accuracy': 0.8406862745098039, 'f1': 0.8794063079777366}\n"824 ]825 },826 {827 "output_type": "stream",828 "name": "stderr",829 "text": [830 "100%|██████████| 459/459 [01:49<00:00, 4.20it/s]\n",831 "100%|██████████| 51/51 [00:06<00:00, 8.45it/s]\n"832 ]833 },834 {835 "output_type": "stream",836 "name": "stdout",837 "text": [838 "epoch 3: {'accuracy': 0.8480392156862745, 'f1': 0.8923611111111109}\n"839 ]840 },841 {842 "output_type": "stream",843 "name": "stderr",844 "text": [845 "100%|██████████| 459/459 [01:49<00:00, 4.18it/s]\n",846 "100%|██████████| 51/51 [00:06<00:00, 7.52it/s]\n"847 ]848 },849 {850 "output_type": "stream",851 "name": "stdout",852 "text": [853 "epoch 4: {'accuracy': 0.8529411764705882, 'f1': 0.894736842105263}\n"854 ]855 },856 {857 "output_type": "stream",858 "name": "stderr",859 "text": [860 "100%|██████████| 459/459 [01:49<00:00, 4.19it/s]\n",861 "100%|██████████| 51/51 [00:06<00:00, 8.47it/s]\n"862 ]863 },864 {865 "output_type": "stream",866 "name": "stdout",867 "text": [868 "epoch 5: {'accuracy': 0.8700980392156863, 'f1': 0.9090909090909091}\n"869 ]870 },871 {872 "output_type": "stream",873 "name": "stderr",874 "text": [875 "100%|██████████| 459/459 [01:49<00:00, 4.20it/s]\n",876 "100%|██████████| 51/51 [00:06<00:00, 8.41it/s]\n"877 ]878 },879 {880 "output_type": "stream",881 "name": "stdout",882 "text": [883 "epoch 6: {'accuracy': 0.875, 'f1': 0.9090909090909091}\n"884 ]885 },886 {887 "output_type": "stream",888 "name": "stderr",889 "text": [890 "100%|██████████| 459/459 [01:49<00:00, 4.21it/s]\n",891 "100%|██████████| 51/51 [00:06<00:00, 8.41it/s]\n"892 ]893 },894 {895 "output_type": "stream",896 "name": "stdout",897 "text": [898 "epoch 7: {'accuracy': 0.8676470588235294, 'f1': 0.9042553191489361}\n"899 ]900 },901 {902 "output_type": "stream",903 "name": "stderr",904 "text": [905 "100%|██████████| 459/459 [01:50<00:00, 4.17it/s]\n",906 "100%|██████████| 51/51 [00:06<00:00, 8.45it/s]\n"907 ]908 },909 {910 "output_type": "stream",911 "name": "stdout",912 "text": [913 "epoch 8: {'accuracy': 0.8700980392156863, 'f1': 0.9068541300527241}\n"914 ]915 },916 {917 "output_type": "stream",918 "name": "stderr",919 "text": [920 "100%|██████████| 459/459 [01:49<00:00, 4.20it/s]\n",921 "100%|██████████| 51/51 [00:06<00:00, 8.39it/s]\n"922 ]923 },924 {925 "output_type": "stream",926 "name": "stdout",927 "text": [928 "epoch 9: {'accuracy': 0.8676470588235294, 'f1': 0.9028776978417268}\n"929 ]930 },931 {932 "output_type": "stream",933 "name": "stderr",934 "text": [935 "100%|██████████| 459/459 [01:49<00:00, 4.20it/s]\n",936 "100%|██████████| 51/51 [00:06<00:00, 8.39it/s]\n"937 ]938 },939 {940 "output_type": "stream",941 "name": "stdout",942 "text": [943 "epoch 10: {'accuracy': 0.875, 'f1': 0.9100529100529101}\n"944 ]945 },946 {947 "output_type": "stream",948 "name": "stderr",949 "text": [950 "100%|██████████| 459/459 [01:49<00:00, 4.18it/s]\n",951 "100%|██████████| 51/51 [00:06<00:00, 8.41it/s]"952 ]953 },954 {955 "output_type": "stream",956 "name": "stdout",957 "text": [958 "epoch 11: {'accuracy': 0.8651960784313726, 'f1': 0.9012567324955117}\n"959 ]960 },961 {962 "output_type": "stream",963 "name": "stderr",964 "text": [965 "\n"966 ]967 }968 ],969 "source": [970 "model.to(device)\n",971 "for epoch in range(num_epochs):\n",972 " model.train()\n",973 " for step, batch in enumerate(tqdm(train_dataloader)):\n",974 " batch.to(device)\n",975 " outputs = model(**batch)\n",976 " loss = outputs.loss\n",977 " loss.backward()\n",978 " optimizer.step()\n",979 " lr_scheduler.step()\n",980 " optimizer.zero_grad()\n",981 "\n",982 " model.eval()\n",983 " for step, batch in enumerate(tqdm(eval_dataloader)):\n",984 " batch.to(device)\n",985 " with torch.no_grad():\n",986 " outputs = model(**batch)\n",987 " predictions = outputs.logits.argmax(dim=-1)\n",988 " predictions, references = predictions, batch[\"labels\"]\n",989 " metric.add_batch(\n",990 " predictions=predictions,\n",991 " references=references,\n",992 " )\n",993 "\n",994 " eval_metric = metric.compute()\n",995 " print(f\"epoch {epoch}:\", eval_metric)"996 ]997 },998 {999 "cell_type": "markdown",1000 "id": "f2b2caca",1001 "metadata": {1002 "id": "f2b2caca"1003 },1004 "source": [1005 "## Share adapters on the 🤗 Hub"1006 ]1007 },1008 {1009 "cell_type": "code",1010 "execution_count": null,1011 "id": "990b3c93",1012 "metadata": {1013 "colab": {1014 "referenced_widgets": [1015 "ad8979af959541b2a5a67f389884f057",1016 "1c020005cfe04de6bdd404ef41e0dfef"1017 ]1018 },1019 "id": "990b3c93",1020 "outputId": "0e2f13c2-0285-4f47-ab9a-d3aefea7ee05"1021 },1022 "outputs": [1023 {1024 "data": {1025 "application/vnd.jupyter.widget-view+json": {1026 "model_id": "ad8979af959541b2a5a67f389884f057",1027 "version_major": 2,1028 "version_minor": 01029 },1030 "text/plain": [1031 "Upload 1 LFS files: 0%| | 0/1 [00:00<?, ?it/s]"1032 ]1033 },1034 "metadata": {},1035 "output_type": "display_data"1036 },1037 {1038 "data": {1039 "application/vnd.jupyter.widget-view+json": {1040 "model_id": "1c020005cfe04de6bdd404ef41e0dfef",1041 "version_major": 2,1042 "version_minor": 01043 },1044 "text/plain": [1045 "adapter_model.bin: 0%| | 0.00/4.93M [00:00<?, ?B/s]"1046 ]1047 },1048 "metadata": {},1049 "output_type": "display_data"1050 },1051 {1052 "data": {1053 "text/plain": [1054 "CommitInfo(commit_url='https://huggingface.co/SumanthRH/roberta-large-peft-ia3/commit/9968de770e53ecd24e8e689a10144b7f55059a75', commit_message='Upload model', commit_description='', oid='9968de770e53ecd24e8e689a10144b7f55059a75', pr_url='https://huggingface.co/SumanthRH/roberta-large-peft-ia3/discussions/3', pr_revision='refs/pr/3', pr_num=3)"1055 ]1056 },1057 "execution_count": 25,1058 "metadata": {},1059 "output_type": "execute_result"1060 }1061 ],1062 "source": [1063 "model.push_to_hub(\"SumanthRH/roberta-large-peft-ia3\", use_auth_token=True)"1064 ]1065 },1066 {1067 "cell_type": "markdown",1068 "id": "9d140b26",1069 "metadata": {1070 "id": "9d140b26"1071 },1072 "source": [1073 "## Load adapters from the Hub\n",1074 "\n",1075 "You can also directly load adapters from the Hub using the commands below:"1076 ]1077 },1078 {1079 "cell_type": "code",1080 "execution_count": null,1081 "id": "4d55c87d",1082 "metadata": {1083 "colab": {1084 "referenced_widgets": [1085 "fd320d7b5d9a43df8abf044897e25cc7",1086 "0022338db24d43bba27d0e72773855e0"1087 ]1088 },1089 "id": "4d55c87d",1090 "outputId": "e6e232d7-22a3-4c85-c72e-90d8275c9762"1091 },1092 "outputs": [1093 {1094 "data": {1095 "application/vnd.jupyter.widget-view+json": {1096 "model_id": "fd320d7b5d9a43df8abf044897e25cc7",1097 "version_major": 2,1098 "version_minor": 01099 },1100 "text/plain": [1101 "Downloading (…)/adapter_config.json: 0%| | 0.00/345 [00:00<?, ?B/s]"1102 ]1103 },1104 "metadata": {},1105 "output_type": "display_data"1106 },1107 {1108 "name": "stderr",1109 "output_type": "stream",1110 "text": [1111 "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'roberta.pooler.dense.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.bias']\n",1112 "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",1113 "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",1114 "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",1115 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"1116 ]1117 },1118 {1119 "data": {1120 "application/vnd.jupyter.widget-view+json": {1121 "model_id": "0022338db24d43bba27d0e72773855e0",1122 "version_major": 2,1123 "version_minor": 01124 },1125 "text/plain": [1126 "Downloading adapter_model.bin: 0%| | 0.00/4.93M [00:00<?, ?B/s]"1127 ]1128 },1129 "metadata": {},1130 "output_type": "display_data"1131 },1132 {1133 "name": "stderr",1134 "output_type": "stream",1135 "text": [1136 " 0%| | 0/51 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",1137 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 51/51 [00:04<00:00, 10.57it/s]"1138 ]1139 },1140 {1141 "name": "stdout",1142 "output_type": "stream",1143 "text": [1144 "{'accuracy': 0.8774509803921569, 'f1': 0.9116607773851589}\n"1145 ]1146 },1147 {1148 "name": "stderr",1149 "output_type": "stream",1150 "text": [1151 "\n"1152 ]1153 }1154 ],1155 "source": [1156 "import torch\n",1157 "from peft import PeftModel, PeftConfig\n",1158 "from transformers import AutoModelForCausalLM, AutoTokenizer\n",1159 "\n",1160 "peft_model_id = \"SumanthRH/roberta-large-peft-ia3\"\n",1161 "config = PeftConfig.from_pretrained(peft_model_id)\n",1162 "inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",1163 "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",1164 "\n",1165 "# Load the Lora model\n",1166 "inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",1167 "\n",1168 "inference_model.to(device)\n",1169 "inference_model.eval()\n",1170 "for step, batch in enumerate(tqdm(eval_dataloader)):\n",1171 " batch.to(device)\n",1172 " with torch.no_grad():\n",1173 " outputs = inference_model(**batch)\n",1174 " predictions = outputs.logits.argmax(dim=-1)\n",1175 " predictions, references = predictions, batch[\"labels\"]\n",1176 " metric.add_batch(\n",1177 " predictions=predictions,\n",1178 " references=references,\n",1179 " )\n",1180 "\n",1181 "eval_metric = metric.compute()\n",1182 "print(eval_metric)"1183 ]1184 },1185 {1186 "cell_type": "code",1187 "execution_count": null,1188 "id": "27c43da1",1189 "metadata": {1190 "id": "27c43da1"1191 },1192 "outputs": [],1193 "source": []1194 }1195 ],1196 "metadata": {1197 "kernelspec": {1198 "display_name": "Python 3",1199 "name": "python3"1200 },