PEFT/conditional-generation
4
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "id": "5f93b7d1",7 "metadata": {8 "ExecuteTime": {9 "end_time": "2023-05-30T08:37:58.711225Z",10 "start_time": "2023-05-30T08:37:56.881307Z"11 }12 },13 "outputs": [14 {15 "name": "stdout",16 "output_type": "stream",17 "text": [18 "\n",19 "===================================BUG REPORT===================================\n",20 "Welcome to bitsandbytes. For bug reports, please run\n",21 "\n",22 "python -m bitsandbytes\n",23 "\n",24 " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",25 "================================================================================\n",26 "bin /udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n",27 "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",28 "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0\n",29 "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",30 "CUDA SETUP: Detected CUDA version 117\n",31 "CUDA SETUP: Loading binary /udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"32 ]33 },34 {35 "name": "stderr",36 "output_type": "stream",37 "text": [38 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /udir/tschilla/anaconda3 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",39 " warn(msg)\n",40 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('Europe/Paris')}\n",41 " warn(msg)\n",42 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/udir/tschilla/.cache/dotnet_bundle_extract')}\n",43 " warn(msg)\n",44 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('5002'), PosixPath('http'), PosixPath('//127.0.0.1')}\n",45 " warn(msg)\n",46 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('() { ( alias;\\n eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@\\n}')}\n",47 " warn(msg)\n",48 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",49 " warn(msg)\n",50 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n",51 "Either way, this might cause trouble in the future:\n",52 "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",53 " warn(msg)\n"54 ]55 }56 ],57 "source": [58 "import os\n",59 "\n",60 "import torch\n",61 "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup\n",62 "from peft import get_peft_model, PromptTuningConfig, TaskType, PromptTuningInit\n",63 "from torch.utils.data import DataLoader\n",64 "from tqdm import tqdm\n",65 "from datasets import load_dataset\n",66 "\n",67 "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",68 "\n",69 "device = \"cuda\"\n",70 "model_name_or_path = \"t5-large\"\n",71 "tokenizer_name_or_path = \"t5-large\"\n",72 "\n",73 "checkpoint_name = \"financial_sentiment_analysis_prompt_tuning_v1.pt\"\n",74 "text_column = \"sentence\"\n",75 "label_column = \"text_label\"\n",76 "max_length = 128\n",77 "lr = 1\n",78 "num_epochs = 5\n",79 "batch_size = 8"80 ]81 },82 {83 "cell_type": "code",84 "execution_count": 2,85 "id": "8d0850ac",86 "metadata": {87 "ExecuteTime": {88 "end_time": "2023-05-30T08:38:12.413984Z",89 "start_time": "2023-05-30T08:38:04.601042Z"90 }91 },92 "outputs": [93 {94 "name": "stdout",95 "output_type": "stream",96 "text": [97 "trainable params: 40960 || all params: 737709056 || trainable%: 0.005552324411210698\n"98 ]99 },100 {101 "name": "stderr",102 "output_type": "stream",103 "text": [104 "/udir/tschilla/anaconda3/envs/peft/lib/python3.9/site-packages/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",105 "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",106 "- Be aware that you SHOULD NOT rely on t5-large automatically truncating your input to 512 when padding/encoding.\n",107 "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",108 "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",109 " warnings.warn(\n"110 ]111 },112 {113 "data": {114 "text/plain": [115 "PeftModelForSeq2SeqLM(\n",116 " (base_model): T5ForConditionalGeneration(\n",117 " (shared): Embedding(32128, 1024)\n",118 " (encoder): T5Stack(\n",119 " (embed_tokens): Embedding(32128, 1024)\n",120 " (block): ModuleList(\n",121 " (0): T5Block(\n",122 " (layer): ModuleList(\n",123 " (0): T5LayerSelfAttention(\n",124 " (SelfAttention): T5Attention(\n",125 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",126 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",127 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",128 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",129 " (relative_attention_bias): Embedding(32, 16)\n",130 " )\n",131 " (layer_norm): T5LayerNorm()\n",132 " (dropout): Dropout(p=0.1, inplace=False)\n",133 " )\n",134 " (1): T5LayerFF(\n",135 " (DenseReluDense): T5DenseActDense(\n",136 " (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",137 " (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",138 " (dropout): Dropout(p=0.1, inplace=False)\n",139 " (act): ReLU()\n",140 " )\n",141 " (layer_norm): T5LayerNorm()\n",142 " (dropout): Dropout(p=0.1, inplace=False)\n",143 " )\n",144 " )\n",145 " )\n",146 " (1-23): 23 x T5Block(\n",147 " (layer): ModuleList(\n",148 " (0): T5LayerSelfAttention(\n",149 " (SelfAttention): T5Attention(\n",150 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",151 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",152 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",153 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",154 " )\n",155 " (layer_norm): T5LayerNorm()\n",156 " (dropout): Dropout(p=0.1, inplace=False)\n",157 " )\n",158 " (1): T5LayerFF(\n",159 " (DenseReluDense): T5DenseActDense(\n",160 " (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",161 " (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",162 " (dropout): Dropout(p=0.1, inplace=False)\n",163 " (act): ReLU()\n",164 " )\n",165 " (layer_norm): T5LayerNorm()\n",166 " (dropout): Dropout(p=0.1, inplace=False)\n",167 " )\n",168 " )\n",169 " )\n",170 " )\n",171 " (final_layer_norm): T5LayerNorm()\n",172 " (dropout): Dropout(p=0.1, inplace=False)\n",173 " )\n",174 " (decoder): T5Stack(\n",175 " (embed_tokens): Embedding(32128, 1024)\n",176 " (block): ModuleList(\n",177 " (0): T5Block(\n",178 " (layer): ModuleList(\n",179 " (0): T5LayerSelfAttention(\n",180 " (SelfAttention): T5Attention(\n",181 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",182 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",183 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",184 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",185 " (relative_attention_bias): Embedding(32, 16)\n",186 " )\n",187 " (layer_norm): T5LayerNorm()\n",188 " (dropout): Dropout(p=0.1, inplace=False)\n",189 " )\n",190 " (1): T5LayerCrossAttention(\n",191 " (EncDecAttention): T5Attention(\n",192 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",193 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",194 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",195 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",196 " )\n",197 " (layer_norm): T5LayerNorm()\n",198 " (dropout): Dropout(p=0.1, inplace=False)\n",199 " )\n",200 " (2): T5LayerFF(\n",201 " (DenseReluDense): T5DenseActDense(\n",202 " (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",203 " (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",204 " (dropout): Dropout(p=0.1, inplace=False)\n",205 " (act): ReLU()\n",206 " )\n",207 " (layer_norm): T5LayerNorm()\n",208 " (dropout): Dropout(p=0.1, inplace=False)\n",209 " )\n",210 " )\n",211 " )\n",212 " (1-23): 23 x T5Block(\n",213 " (layer): ModuleList(\n",214 " (0): T5LayerSelfAttention(\n",215 " (SelfAttention): T5Attention(\n",216 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",217 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",218 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",219 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",220 " )\n",221 " (layer_norm): T5LayerNorm()\n",222 " (dropout): Dropout(p=0.1, inplace=False)\n",223 " )\n",224 " (1): T5LayerCrossAttention(\n",225 " (EncDecAttention): T5Attention(\n",226 " (q): Linear(in_features=1024, out_features=1024, bias=False)\n",227 " (k): Linear(in_features=1024, out_features=1024, bias=False)\n",228 " (v): Linear(in_features=1024, out_features=1024, bias=False)\n",229 " (o): Linear(in_features=1024, out_features=1024, bias=False)\n",230 " )\n",231 " (layer_norm): T5LayerNorm()\n",232 " (dropout): Dropout(p=0.1, inplace=False)\n",233 " )\n",234 " (2): T5LayerFF(\n",235 " (DenseReluDense): T5DenseActDense(\n",236 " (wi): Linear(in_features=1024, out_features=4096, bias=False)\n",237 " (wo): Linear(in_features=4096, out_features=1024, bias=False)\n",238 " (dropout): Dropout(p=0.1, inplace=False)\n",239 " (act): ReLU()\n",240 " )\n",241 " (layer_norm): T5LayerNorm()\n",242 " (dropout): Dropout(p=0.1, inplace=False)\n",243 " )\n",244 " )\n",245 " )\n",246 " )\n",247 " (final_layer_norm): T5LayerNorm()\n",248 " (dropout): Dropout(p=0.1, inplace=False)\n",249 " )\n",250 " (lm_head): Linear(in_features=1024, out_features=32128, bias=False)\n",251 " )\n",252 " (prompt_encoder): ModuleDict(\n",253 " (default): PromptEmbedding(\n",254 " (embedding): Embedding(40, 1024)\n",255 " )\n",256 " )\n",257 " (word_embeddings): Embedding(32128, 1024)\n",258 ")"259 ]260 },261 "execution_count": 2,262 "metadata": {},263 "output_type": "execute_result"264 }265 ],266 "source": [267 "# creating model\n",268 "peft_config = PromptTuningConfig(\n",269 " task_type=TaskType.SEQ_2_SEQ_LM,\n",270 " prompt_tuning_init=PromptTuningInit.TEXT,\n",271 " num_virtual_tokens=20,\n",272 " prompt_tuning_init_text=\"What is the sentiment of this article?\\n\",\n",273 " inference_mode=False,\n",274 " tokenizer_name_or_path=model_name_or_path,\n",275 ")\n",276 "\n",277 "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",278 "model = get_peft_model(model, peft_config)\n",279 "model.print_trainable_parameters()\n",280 "model"281 ]282 },283 {284 "cell_type": "code",285 "execution_count": 3,286 "id": "4ee2babf",287 "metadata": {288 "ExecuteTime": {289 "end_time": "2023-05-30T08:38:18.759143Z",290 "start_time": "2023-05-30T08:38:17.881621Z"291 }292 },293 "outputs": [294 {295 "name": "stderr",296 "output_type": "stream",297 "text": [298 "Found cached dataset financial_phrasebank (/data/proxem/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n"299 ]300 },301 {302 "data": {303 "application/vnd.jupyter.widget-view+json": {304 "model_id": "fb63f50cb7cb4f5aae10648ba74d6c4e",305 "version_major": 2,306 "version_minor": 0307 },308 "text/plain": [309 " 0%| | 0/1 [00:00<?, ?it/s]"310 ]311 },312 "metadata": {},313 "output_type": "display_data"314 },315 {316 "data": {317 "application/vnd.jupyter.widget-view+json": {318 "model_id": "",319 "version_major": 2,320 "version_minor": 0321 },322 "text/plain": [323 "Map: 0%| | 0/2037 [00:00<?, ? examples/s]"324 ]325 },326 "metadata": {},327 "output_type": "display_data"328 },329 {330 "data": {331 "application/vnd.jupyter.widget-view+json": {332 "model_id": "",333 "version_major": 2,334 "version_minor": 0335 },336 "text/plain": [337 "Map: 0%| | 0/227 [00:00<?, ? examples/s]"338 ]339 },340 "metadata": {},341 "output_type": "display_data"342 },343 {344 "data": {345 "text/plain": [346 "{'sentence': '`` Lining stone sales were also good in the early autumn , and order books are strong to the end of the year .',\n",347 " 'label': 2,\n",348 " 'text_label': 'positive'}"349 ]350 },351 "execution_count": 3,352 "metadata": {},353 "output_type": "execute_result"354 }355 ],356 "source": [357 "# loading dataset\n",358 "dataset = load_dataset(\"financial_phrasebank\", \"sentences_allagree\")\n",359 "dataset = dataset[\"train\"].train_test_split(test_size=0.1)\n",360 "dataset[\"validation\"] = dataset[\"test\"]\n",361 "del dataset[\"test\"]\n",362 "\n",363 "classes = dataset[\"train\"].features[\"label\"].names\n",364 "dataset = dataset.map(\n",365 " lambda x: {\"text_label\": [classes[label] for label in x[\"label\"]]},\n",366 " batched=True,\n",367 " num_proc=1,\n",368 ")\n",369 "\n",370 "dataset[\"train\"][0]"371 ]372 },373 {374 "cell_type": "code",375 "execution_count": 4,376 "id": "adf9608c",377 "metadata": {378 "ExecuteTime": {379 "end_time": "2023-05-30T08:38:21.132266Z",380 "start_time": "2023-05-30T08:38:20.340722Z"381 }382 },383 "outputs": [384 {385 "data": {386 "application/vnd.jupyter.widget-view+json": {387 "model_id": "",388 "version_major": 2,389 "version_minor": 0390 },391 "text/plain": [392 "Running tokenizer on dataset: 0%| | 0/2037 [00:00<?, ? examples/s]"393 ]394 },395 "metadata": {},396 "output_type": "display_data"397 },398 {399 "data": {400 "application/vnd.jupyter.widget-view+json": {401 "model_id": "",402 "version_major": 2,403 "version_minor": 0404 },405 "text/plain": [406 "Running tokenizer on dataset: 0%| | 0/227 [00:00<?, ? examples/s]"407 ]408 },409 "metadata": {},410 "output_type": "display_data"411 }412 ],413 "source": [414 "# data preprocessing\n",415 "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",416 "target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n",417 "\n",418 "\n",419 "def preprocess_function(examples):\n",420 " inputs = examples[text_column]\n",421 " targets = examples[label_column]\n",422 " model_inputs = tokenizer(inputs, max_length=max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",423 " labels = tokenizer(\n",424 " targets, max_length=target_max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\"\n",425 " )\n",426 " labels = labels[\"input_ids\"]\n",427 " labels[labels == tokenizer.pad_token_id] = -100\n",428 " model_inputs[\"labels\"] = labels\n",429 " return model_inputs\n",430 "\n",431 "\n",432 "processed_datasets = dataset.map(\n",433 " preprocess_function,\n",434 " batched=True,\n",435 " num_proc=1,\n",436 " remove_columns=dataset[\"train\"].column_names,\n",437 " load_from_cache_file=False,\n",438 " desc=\"Running tokenizer on dataset\",\n",439 ")\n",440 "\n",441 "train_dataset = processed_datasets[\"train\"]\n",442 "eval_dataset = processed_datasets[\"validation\"]\n",443 "\n",444 "train_dataloader = DataLoader(\n",445 " train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",446 ")\n",447 "eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"448 ]449 },450 {451 "cell_type": "code",452 "execution_count": 5,453 "id": "f733a3c6",454 "metadata": {455 "ExecuteTime": {456 "end_time": "2023-05-30T08:38:22.907922Z",457 "start_time": "2023-05-30T08:38:22.901057Z"458 }459 },460 "outputs": [],461 "source": [462 "# optimizer and lr scheduler\n",463 "optimizer = torch.optim.AdamW(model.parameters(), lr=lr)\n",464 "lr_scheduler = get_linear_schedule_with_warmup(\n",465 " optimizer=optimizer,\n",466 " num_warmup_steps=0,\n",467 " num_training_steps=(len(train_dataloader) * num_epochs),\n",468 ")"469 ]470 },471 {472 "cell_type": "code",473 "execution_count": 7,474 "id": "6b3a4090",475 "metadata": {476 "ExecuteTime": {477 "end_time": "2023-05-30T08:42:29.409070Z",478 "start_time": "2023-05-30T08:38:50.102263Z"479 }480 },481 "outputs": [482 {483 "name": "stderr",484 "output_type": "stream",485 "text": [486 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 255/255 [00:42<00:00, 6.05it/s]\n",487 "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.40it/s]\n"488 ]489 },490 {491 "name": "stdout",492 "output_type": "stream",493 "text": [494 "epoch=0: train_ppl=tensor(8.0846, device='cuda:0') train_epoch_loss=tensor(2.0900, device='cuda:0') eval_ppl=tensor(1.3542, device='cuda:0') eval_epoch_loss=tensor(0.3032, device='cuda:0')\n"495 ]496 },497 {498 "name": "stderr",499 "output_type": "stream",500 "text": [501 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 255/255 [00:41<00:00, 6.15it/s]\n",502 "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.42it/s]\n"503 ]504 },505 {506 "name": "stdout",507 "output_type": "stream",508 "text": [509 "epoch=1: train_ppl=tensor(1.5088, device='cuda:0') train_epoch_loss=tensor(0.4113, device='cuda:0') eval_ppl=tensor(1.2692, device='cuda:0') eval_epoch_loss=tensor(0.2384, device='cuda:0')\n"510 ]511 },512 {513 "name": "stderr",514 "output_type": "stream",515 "text": [516 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 255/255 [00:41<00:00, 6.18it/s]\n",517 "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.45it/s]\n"518 ]519 },520 {521 "name": "stdout",522 "output_type": "stream",523 "text": [524 "epoch=2: train_ppl=tensor(1.5322, device='cuda:0') train_epoch_loss=tensor(0.4267, device='cuda:0') eval_ppl=tensor(1.2065, device='cuda:0') eval_epoch_loss=tensor(0.1877, device='cuda:0')\n"525 ]526 },527 {528 "name": "stderr",529 "output_type": "stream",530 "text": [531 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 255/255 [00:41<00:00, 6.17it/s]\n",532 "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.38it/s]\n"533 ]534 },535 {536 "name": "stdout",537 "output_type": "stream",538 "text": [539 "epoch=3: train_ppl=tensor(1.4475, device='cuda:0') train_epoch_loss=tensor(0.3699, device='cuda:0') eval_ppl=tensor(1.2346, device='cuda:0') eval_epoch_loss=tensor(0.2107, device='cuda:0')\n"540 ]541 },542 {543 "name": "stderr",544 "output_type": "stream",545 "text": [546 "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 255/255 [00:42<00:00, 5.94it/s]\n",547 "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.42it/s]"548 ]549 },550 {551 "name": "stdout",552 "output_type": "stream",553 "text": [554 "epoch=4: train_ppl=tensor(1.3428, device='cuda:0') train_epoch_loss=tensor(0.2948, device='cuda:0') eval_ppl=tensor(1.2041, device='cuda:0') eval_epoch_loss=tensor(0.1857, device='cuda:0')\n"555 ]556 },557 {558 "name": "stderr",559 "output_type": "stream",560 "text": [561 "\n"562 ]563 }564 ],565 "source": [566 "# training and evaluation\n",567 "model = model.to(device)\n",568 "\n",569 "for epoch in range(num_epochs):\n",570 " model.train()\n",571 " total_loss = 0\n",572 " for step, batch in enumerate(tqdm(train_dataloader)):\n",573 " batch = {k: v.to(device) for k, v in batch.items()}\n",574 " outputs = model(**batch)\n",575 " loss = outputs.loss\n",576 " total_loss += loss.detach().float()\n",577 " loss.backward()\n",578 " optimizer.step()\n",579 " lr_scheduler.step()\n",580 " optimizer.zero_grad()\n",581 "\n",582 " model.eval()\n",583 " eval_loss = 0\n",584 " eval_preds = []\n",585 " for step, batch in enumerate(tqdm(eval_dataloader)):\n",586 " batch = {k: v.to(device) for k, v in batch.items()}\n",587 " with torch.no_grad():\n",588 " outputs = model(**batch)\n",589 " loss = outputs.loss\n",590 " eval_loss += loss.detach().float()\n",591 " eval_preds.extend(\n",592 " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",593 " )\n",594 "\n",595 " eval_epoch_loss = eval_loss / len(eval_dataloader)\n",596 " eval_ppl = torch.exp(eval_epoch_loss)\n",597 " train_epoch_loss = total_loss / len(train_dataloader)\n",598 " train_ppl = torch.exp(train_epoch_loss)\n",599 " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"600 ]601 },602 {603 "cell_type": "code",604 "execution_count": 8,605 "id": "6cafa67b",606 "metadata": {607 "ExecuteTime": {608 "end_time": "2023-05-30T08:42:42.844671Z",609 "start_time": "2023-05-30T08:42:42.840447Z"610 }611 },612 "outputs": [613 {614 "name": "stdout",615 "output_type": "stream",616 "text": [617 "accuracy=85.46255506607929 % on the evaluation dataset\n",618 "eval_preds[:10]=['neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'positive', 'neutral', 'negative', 'neutral', 'positive']\n",619 "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'positive', 'neutral', 'negative', 'positive', 'neutral']\n"620 ]621 }622 ],623 "source": [624 "# print accuracy\n",625 "correct = 0\n",626 "total = 0\n",627 "for pred, true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",628 " if pred.strip() == true.strip():\n",629 " correct += 1\n",630 " total += 1\n",631 "accuracy = correct / total * 100\n",632 "print(f\"{accuracy=} % on the evaluation dataset\")\n",633 "print(f\"{eval_preds[:10]=}\")\n",634 "print(f\"{dataset['validation']['text_label'][:10]=}\")"635 ]636 },637 {638 "cell_type": "code",639 "execution_count": 9,640 "id": "a8de6005",641 "metadata": {642 "ExecuteTime": {643 "end_time": "2023-05-30T08:42:45.752765Z",644 "start_time": "2023-05-30T08:42:45.742397Z"645 }646 },647 "outputs": [],648 "source": [649 "# saving model\n",650 "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",651 "model.save_pretrained(peft_model_id)"652 ]653 },654 {655 "cell_type": "code",656 "execution_count": 10,657 "id": "bd20cd4c",658 "metadata": {659 "ExecuteTime": {660 "end_time": "2023-05-30T08:42:47.660873Z",661 "start_time": "2023-05-30T08:42:47.488293Z"662 }663 },664 "outputs": [665 {666 "name": "stdout",667 "output_type": "stream",668 "text": [669 "164K\tt5-large_PROMPT_TUNING_SEQ_2_SEQ_LM/adapter_model.bin\r\n"670 ]671 }672 ],673 "source": [674 "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n",675 "!du -h $ckpt"676 ]677 },678 {679 "cell_type": "code",680 "execution_count": 11,681 "id": "76c2fc29",682 "metadata": {683 "ExecuteTime": {684 "end_time": "2023-05-30T08:42:56.721990Z",685 "start_time": "2023-05-30T08:42:49.060700Z"686 }687 },688 "outputs": [],689 "source": [690 "from peft import PeftModel, PeftConfig\n",691 "\n",692 "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",693 "\n",694 "config = PeftConfig.from_pretrained(peft_model_id)\n",695 "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n",696 "model = PeftModel.from_pretrained(model, peft_model_id)"697 ]698 },699 {700 "cell_type": "code",701 "execution_count": 12,702 "id": "d997f1cc",703 "metadata": {704 "ExecuteTime": {705 "end_time": "2023-05-30T08:42:59.600916Z",706 "start_time": "2023-05-30T08:42:58.961468Z"707 }708 },709 "outputs": [710 {711 "name": "stdout",712 "output_type": "stream",713 "text": [714 "Danske Bank is Denmark 's largest bank with 3.5 million customers .\n",715 "tensor([[ 3039, 1050, 1925, 19, 18001, 3, 31, 7, 2015, 2137,\n",716 " 28, 3, 9285, 770, 722, 3, 5, 1]])\n",717 "tensor([[ 0, 7163, 1]])\n",718 "['neutral']\n"719 ]720 }721 ],722 "source": [723 "model.eval()\n",724 "i = 107\n",725 "input_ids = tokenizer(dataset[\"validation\"][text_column][i], return_tensors=\"pt\").input_ids\n",726 "print(dataset[\"validation\"][text_column][i])\n",727 "print(input_ids)\n",728 "\n",729 "with torch.no_grad():\n",730 " outputs = model.generate(input_ids=input_ids, max_new_tokens=10)\n",731 " print(outputs)\n",732 " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"733 ]734 }735 ],736 "metadata": {737 "kernelspec": {738 "display_name": "peft",739 "language": "python",740 "name": "peft"741 },742 "language_info": {743 "codemirror_mode": {744 "name": "ipython",745 "version": 3746 },747 "file_extension": ".py",748 "mimetype": "text/x-python",749 "name": "python",750 "nbconvert_exporter": "python",751 "pygments_lexer": "ipython3",752 "version": "3.9.16"753 },754 "toc": {755 "base_numbering": 1,756 "nav_menu": {},757 "number_sections": true,758 "sideBar": true,759 "skip_h1_title": false,760 "title_cell": "Table of Contents",761 "title_sidebar": "Contents",762 "toc_cell": false,763 "toc_position": {},764 "toc_section_display": true,765 "toc_window_display": false766 },767 "varInspector": {768 "cols": {769 "lenName": 16,770 "lenType": 16,771 "lenVar": 40772 },773 "kernels_config": {774 "python": {775 "delete_cmd_postfix": "",776 "delete_cmd_prefix": "del ",777 "library": "var_list.py",778 "varRefreshCmd": "print(var_dic_list())"779 },780 "r": {781 "delete_cmd_postfix": ") ",782 "delete_cmd_prefix": "rm(",783 "library": "var_list.r",784 "varRefreshCmd": "cat(var_dic_list()) "785 }786 },787 "types_to_exclude": [788 "module",789 "function",790 "builtin_function_or_method",791 "instance",792 "_Feature"793 ],794 "window_display": false795 },796 "vscode": {797 "interpreter": {798 "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"799 }800 }801 },802 "nbformat": 4,803 "nbformat_minor": 5804}805