Sarath2002/Form_Understanding_using_LayoutLMV3
1
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "metadata": {7 "dotnet_interactive": {8 "language": "csharp"9 },10 "polyglot_notebook": {11 "kernelName": "csharp"12 }13 },14 "outputs": [15 {16 "name": "stderr",17 "output_type": "stream",18 "text": [19 "g:\\IDEs and Modules\\Anaconda\\envs\\pytorch_gpu\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",20 " from .autonotebook import tqdm as notebook_tqdm\n",21 "Found cached dataset funsd-layoutlmv3 (C:/Users/csara/.cache/huggingface/datasets/nielsr___funsd-layoutlmv3/funsd/1.0.0/0e3f4efdfd59aa1c3b4952c517894f7b1fc4d75c12ef01bcc8626a69e41c1bb9)\n",22 "100%|██████████| 2/2 [00:00<00:00, 290.08it/s]\n"23 ]24 }25 ],26 "source": [27 "from datasets import load_dataset \n",28 "dataset = load_dataset(\"nielsr/funsd-layoutlmv3\")"29 ]30 },31 {32 "cell_type": "code",33 "execution_count": 2,34 "metadata": {35 "execution": {36 "iopub.execute_input": "2023-03-03T19:12:08.141445Z",37 "iopub.status.busy": "2023-03-03T19:12:08.140739Z",38 "iopub.status.idle": "2023-03-03T19:12:24.061622Z",39 "shell.execute_reply": "2023-03-03T19:12:24.060499Z",40 "shell.execute_reply.started": "2023-03-03T19:12:08.141404Z"41 },42 "trusted": true43 },44 "outputs": [],45 "source": [46 "from transformers import AutoProcessor\n",47 "from datasets.features import ClassLabel\n",48 "tokenizer = AutoProcessor.from_pretrained(\"microsoft/layoutlmv3-base\", apply_ocr=False)"49 ]50 },51 {52 "cell_type": "code",53 "execution_count": 3,54 "metadata": {55 "execution": {56 "iopub.execute_input": "2023-03-03T19:12:49.955307Z",57 "iopub.status.busy": "2023-03-03T19:12:49.954390Z",58 "iopub.status.idle": "2023-03-03T19:12:49.962064Z",59 "shell.execute_reply": "2023-03-03T19:12:49.961038Z",60 "shell.execute_reply.started": "2023-03-03T19:12:49.955256Z"61 },62 "trusted": true63 },64 "outputs": [],65 "source": [66 "def get_label_list(labels):\n",67 " unique_labels = set()\n",68 " for label in labels:\n",69 " unique_labels = unique_labels | set(label)\n",70 " label_list = list(unique_labels)\n",71 " label_list.sort()\n",72 " return label_list"73 ]74 },75 {76 "cell_type": "code",77 "execution_count": 4,78 "metadata": {79 "execution": {80 "iopub.execute_input": "2023-03-03T19:12:59.478905Z",81 "iopub.status.busy": "2023-03-03T19:12:59.478538Z",82 "iopub.status.idle": "2023-03-03T19:12:59.487011Z",83 "shell.execute_reply": "2023-03-03T19:12:59.485535Z",84 "shell.execute_reply.started": "2023-03-03T19:12:59.478872Z"85 },86 "trusted": true87 },88 "outputs": [],89 "source": [90 "image_column_name = \"image\"\n",91 "text_column_name = \"tokens\"\n",92 "boxes_column_name = \"bboxes\"\n",93 "label_column_name = \"ner_tags\"\n",94 "\n",95 "\n",96 "features = dataset[\"train\"].features\n",97 "column_names = dataset[\"train\"].column_names\n",98 "\n",99 "if isinstance(features[\"ner_tags\"].feature, ClassLabel):\n",100 " label_list = features[\"ner_tags\"].feature.names\n",101 " id2label = {k: v for k,v in enumerate(label_list)}\n",102 " label2id = {v: k for k,v in enumerate(label_list)}\n",103 "else:\n",104 " label_list = get_label_list(dataset[\"train\"][\"ner_tags\"])\n",105 " id2label = {k: v for k,v in enumerate(label_list)}\n",106 " label2id = {v: k for k,v in enumerate(label_list)}\n",107 "num_labels = len(label_list)"108 ]109 },110 {111 "cell_type": "code",112 "execution_count": 5,113 "metadata": {114 "execution": {115 "iopub.execute_input": "2023-03-03T19:13:09.900559Z",116 "iopub.status.busy": "2023-03-03T19:13:09.900179Z",117 "iopub.status.idle": "2023-03-03T19:13:09.907248Z",118 "shell.execute_reply": "2023-03-03T19:13:09.906077Z",119 "shell.execute_reply.started": "2023-03-03T19:13:09.900526Z"120 },121 "trusted": true122 },123 "outputs": [],124 "source": [125 "def encoder(examples):\n",126 " images = examples[\"image\"]\n",127 " words = examples[\"tokens\"]\n",128 " boxes = examples[\"bboxes\"]\n",129 " word_labels = examples[label_column_name]\n",130 "\n",131 " encoding = tokenizer(images, words, boxes=boxes, word_labels=word_labels,\n",132 " truncation=True, padding=\"max_length\")\n",133 "\n",134 " return encoding"135 ]136 },137 {138 "cell_type": "code",139 "execution_count": 6,140 "metadata": {141 "execution": {142 "iopub.execute_input": "2023-03-03T19:13:22.537584Z",143 "iopub.status.busy": "2023-03-03T19:13:22.537226Z",144 "iopub.status.idle": "2023-03-03T19:13:28.944323Z",145 "shell.execute_reply": "2023-03-03T19:13:28.943322Z",146 "shell.execute_reply.started": "2023-03-03T19:13:22.537552Z"147 },148 "trusted": true149 },150 "outputs": [151 {152 "name": "stderr",153 "output_type": "stream",154 "text": [155 "Loading cached processed dataset at C:\\Users\\csara\\.cache\\huggingface\\datasets\\nielsr___funsd-layoutlmv3\\funsd\\1.0.0\\0e3f4efdfd59aa1c3b4952c517894f7b1fc4d75c12ef01bcc8626a69e41c1bb9\\cache-1a2006e093366773.arrow\n",156 "Loading cached processed dataset at C:\\Users\\csara\\.cache\\huggingface\\datasets\\nielsr___funsd-layoutlmv3\\funsd\\1.0.0\\0e3f4efdfd59aa1c3b4952c517894f7b1fc4d75c12ef01bcc8626a69e41c1bb9\\cache-b7746bd565a79622.arrow\n"157 ]158 }159 ],160 "source": [161 "from datasets import Features, Sequence, ClassLabel, Value, Array2D, Array3D\n",162 "\n",163 "\n",164 "features = Features({\n",165 " 'pixel_values': Array3D(dtype=\"float32\", shape=(3, 224, 224)),\n",166 " 'input_ids': Sequence(feature=Value(dtype='int64')),\n",167 " 'attention_mask': Sequence(Value(dtype='int64')),\n",168 " 'bbox': Array2D(dtype=\"int64\", shape=(512, 4)),\n",169 " 'labels': Sequence(feature=Value(dtype='int64')),\n",170 "})\n",171 "\n",172 "train_dataset = dataset[\"train\"].map(\n",173 " encoder,\n",174 " batched=True,\n",175 " remove_columns=column_names,\n",176 " features=features,\n",177 ")\n",178 "eval_dataset = dataset[\"test\"].map(\n",179 " encoder,\n",180 " batched=True,\n",181 " remove_columns=column_names,\n",182 " features=features,\n",183 ")"184 ]185 },186 {187 "cell_type": "code",188 "execution_count": 7,189 "metadata": {190 "execution": {191 "iopub.execute_input": "2023-03-03T19:13:40.875858Z",192 "iopub.status.busy": "2023-03-03T19:13:40.875401Z",193 "iopub.status.idle": "2023-03-03T19:13:40.882514Z",194 "shell.execute_reply": "2023-03-03T19:13:40.881313Z",195 "shell.execute_reply.started": "2023-03-03T19:13:40.875816Z"196 },197 "trusted": true198 },199 "outputs": [],200 "source": [201 "train_dataset.set_format(\"torch\")\n"202 ]203 },204 {205 "cell_type": "code",206 "execution_count": 3,207 "metadata": {208 "execution": {209 "iopub.execute_input": "2023-03-03T19:13:47.654814Z",210 "iopub.status.busy": "2023-03-03T19:13:47.654437Z",211 "iopub.status.idle": "2023-03-03T19:13:47.691746Z",212 "shell.execute_reply": "2023-03-03T19:13:47.690779Z",213 "shell.execute_reply.started": "2023-03-03T19:13:47.654780Z"214 },215 "trusted": true216 },217 "outputs": [],218 "source": [219 "import torch"220 ]221 },222 {223 "cell_type": "code",224 "execution_count": 9,225 "metadata": {226 "execution": {227 "iopub.execute_input": "2023-03-03T19:16:13.499216Z",228 "iopub.status.busy": "2023-03-03T19:16:13.498129Z",229 "iopub.status.idle": "2023-03-03T19:16:14.186549Z",230 "shell.execute_reply": "2023-03-03T19:16:14.185533Z",231 "shell.execute_reply.started": "2023-03-03T19:16:13.499170Z"232 },233 "trusted": true234 },235 "outputs": [236 {237 "name": "stderr",238 "output_type": "stream",239 "text": [240 "C:\\Users\\csara\\AppData\\Local\\Temp\\ipykernel_21636\\3673091550.py:2: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",241 " metric = load_metric(\"seqeval\")\n"242 ]243 }244 ],245 "source": [246 "from datasets import load_metric\n",247 "metric = load_metric(\"seqeval\")\n",248 "\n",249 "import numpy as np\n",250 "\n",251 "return_entity_level_metrics = False\n",252 "\n",253 "def compute_metrics(p):\n",254 " predictions, labels = p\n",255 " predictions = np.argmax(predictions, axis=2)\n",256 "\n",257 " # Remove ignored index (special tokens)\n",258 " true_predictions = [\n",259 " [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",260 " for prediction, label in zip(predictions, labels)\n",261 " ]\n",262 " true_labels = [\n",263 " [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",264 " for prediction, label in zip(predictions, labels)\n",265 " ]\n",266 "\n",267 " results = metric.compute(predictions=true_predictions, references=true_labels)\n",268 " if return_entity_level_metrics:\n",269 " # Unpack nested dictionaries\n",270 " final_results = {}\n",271 " for key, value in results.items():\n",272 " if isinstance(value, dict):\n",273 " for n, v in value.items():\n",274 " final_results[f\"{key}_{n}\"] = v\n",275 " else:\n",276 " final_results[key] = value\n",277 " return final_results\n",278 " else:\n",279 " return {\n",280 " \"precision\": results[\"overall_precision\"],\n",281 " \"recall\": results[\"overall_recall\"],\n",282 " \"f1\": results[\"overall_f1\"],\n",283 " \"accuracy\": results[\"overall_accuracy\"],\n",284 " }\n"285 ]286 },287 {288 "cell_type": "code",289 "execution_count": 10,290 "metadata": {291 "execution": {292 "iopub.execute_input": "2023-03-03T19:16:23.761664Z",293 "iopub.status.busy": "2023-03-03T19:16:23.760963Z",294 "iopub.status.idle": "2023-03-03T19:16:28.559914Z",295 "shell.execute_reply": "2023-03-03T19:16:28.558886Z",296 "shell.execute_reply.started": "2023-03-03T19:16:23.761620Z"297 },298 "trusted": true299 },300 "outputs": [301 {302 "name": "stderr",303 "output_type": "stream",304 "text": [305 "Some weights of LayoutLMv3ForTokenClassification were not initialized from the model checkpoint at microsoft/layoutlmv3-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",306 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"307 ]308 }309 ],310 "source": [311 "from transformers import LayoutLMv3ForTokenClassification\n",312 "\n",313 "model = LayoutLMv3ForTokenClassification.from_pretrained(\"microsoft/layoutlmv3-base\",\n",314 " id2label=id2label,\n",315 " label2id=label2id)"316 ]317 },318 {319 "cell_type": "code",320 "execution_count": 11,321 "metadata": {322 "execution": {323 "iopub.execute_input": "2023-03-03T19:16:31.406847Z",324 "iopub.status.busy": "2023-03-03T19:16:31.405450Z",325 "iopub.status.idle": "2023-03-03T19:16:31.608864Z",326 "shell.execute_reply": "2023-03-03T19:16:31.607848Z",327 "shell.execute_reply.started": "2023-03-03T19:16:31.406796Z"328 },329 "trusted": true330 },331 "outputs": [],332 "source": [333 "from transformers import TrainingArguments, Trainer\n",334 "\n",335 "training_args = TrainingArguments(output_dir=\"test\",\n",336 " max_steps=1000,\n",337 " per_device_train_batch_size=2,\n",338 " per_device_eval_batch_size=3,\n",339 " learning_rate=1e-5,\n",340 " evaluation_strategy=\"steps\",\n",341 " eval_steps=100,\n",342 " load_best_model_at_end=True,\n",343 " metric_for_best_model=\"f1\")"344 ]345 },346 {347 "cell_type": "code",348 "execution_count": 12,349 "metadata": {350 "execution": {351 "iopub.execute_input": "2023-03-03T19:16:33.355517Z",352 "iopub.status.busy": "2023-03-03T19:16:33.354801Z",353 "iopub.status.idle": "2023-03-03T19:16:38.092937Z",354 "shell.execute_reply": "2023-03-03T19:16:38.091904Z",355 "shell.execute_reply.started": "2023-03-03T19:16:33.355477Z"356 },357 "trusted": true358 },359 "outputs": [],360 "source": [361 "from transformers.data.data_collator import default_data_collator\n",362 "\n",363 "# Initialize our Trainer\n",364 "trainer = Trainer(\n",365 " model=model,\n",366 " args=training_args,\n",367 " train_dataset=train_dataset,\n",368 " eval_dataset=eval_dataset,\n",369 " tokenizer=tokenizer,\n",370 " data_collator=default_data_collator,\n",371 " compute_metrics=compute_metrics,\n",372 ")"373 ]374 },375 {376 "cell_type": "code",377 "execution_count": 13,378 "metadata": {379 "execution": {380 "iopub.execute_input": "2023-03-03T19:16:40.470684Z",381 "iopub.status.busy": "2023-03-03T19:16:40.469983Z",382 "iopub.status.idle": "2023-03-03T19:23:15.995022Z",383 "shell.execute_reply": "2023-03-03T19:23:15.993467Z",384 "shell.execute_reply.started": "2023-03-03T19:16:40.470647Z"385 },386 "trusted": true387 },388 "outputs": [389 {390 "name": "stderr",391 "output_type": "stream",392 "text": [393 "g:\\IDEs and Modules\\Anaconda\\envs\\pytorch_gpu\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",394 " warnings.warn(\n",395 " 0%| | 0/1000 [00:00<?, ?it/s]g:\\IDEs and Modules\\Anaconda\\envs\\pytorch_gpu\\lib\\site-packages\\transformers\\modeling_utils.py:884: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers.\n",396 " warnings.warn(\n",397 " \n",398 " 10%|█ | 100/1000 [05:03<46:20, 3.09s/it]"399 ]400 },401 {402 "name": "stdout",403 "output_type": "stream",404 "text": [405 "{'eval_loss': 0.6499422192573547, 'eval_precision': 0.7698232895333031, 'eval_recall': 0.8440139095876801, 'eval_f1': 0.8052132701421801, 'eval_accuracy': 0.8016165458219422, 'eval_runtime': 34.3294, 'eval_samples_per_second': 1.456, 'eval_steps_per_second': 0.495, 'epoch': 1.33}\n"406 ]407 },408 {409 "name": "stderr",410 "output_type": "stream",411 "text": [412 " \n",413 " 20%|██ | 200/1000 [11:12<38:47, 2.91s/it]"414 ]415 },416 {417 "name": "stdout",418 "output_type": "stream",419 "text": [420 "{'eval_loss': 0.4936561584472656, 'eval_precision': 0.8225578102878717, 'eval_recall': 0.8658718330849479, 'eval_f1': 0.8436592449177153, 'eval_accuracy': 0.8246760965172947, 'eval_runtime': 35.9544, 'eval_samples_per_second': 1.391, 'eval_steps_per_second': 0.473, 'epoch': 2.67}\n"421 ]422 },423 {424 "name": "stderr",425 "output_type": "stream",426 "text": [427 " \n",428 " 30%|███ | 300/1000 [17:00<33:27, 2.87s/it]"429 ]430 },431 {432 "name": "stdout",433 "output_type": "stream",434 "text": [435 "{'eval_loss': 0.4679512083530426, 'eval_precision': 0.8520765282314512, 'eval_recall': 0.907103825136612, 'eval_f1': 0.8787295476419633, 'eval_accuracy': 0.8576013312730298, 'eval_runtime': 35.098, 'eval_samples_per_second': 1.425, 'eval_steps_per_second': 0.484, 'epoch': 4.0}\n"436 ]437 },438 {439 "name": "stderr",440 "output_type": "stream",441 "text": [442 " \n",443 " 40%|████ | 400/1000 [22:50<29:56, 2.99s/it]"444 ]445 },446 {447 "name": "stdout",448 "output_type": "stream",449 "text": [450 "{'eval_loss': 0.5078234076499939, 'eval_precision': 0.8813806514341274, 'eval_recall': 0.9006458022851466, 'eval_f1': 0.890909090909091, 'eval_accuracy': 0.849994056816831, 'eval_runtime': 38.4245, 'eval_samples_per_second': 1.301, 'eval_steps_per_second': 0.442, 'epoch': 5.33}\n"451 ]452 },453 {454 "name": "stderr",455 "output_type": "stream",456 "text": [457 " 50%|█████ | 500/1000 [28:23<25:09, 3.02s/it] "458 ]459 },460 {461 "name": "stdout",462 "output_type": "stream",463 "text": [464 "{'loss': 0.5405, 'learning_rate': 5e-06, 'epoch': 6.67}\n"465 ]466 },467 {468 "name": "stderr",469 "output_type": "stream",470 "text": [471 " \n",472 " 50%|█████ | 500/1000 [29:02<25:09, 3.02s/it]"473 ]474 },475 {476 "name": "stdout",477 "output_type": "stream",478 "text": [479 "{'eval_loss': 0.5313129425048828, 'eval_precision': 0.8702807357212003, 'eval_recall': 0.8931942374565326, 'eval_f1': 0.8815886246629075, 'eval_accuracy': 0.8495186021633186, 'eval_runtime': 37.6242, 'eval_samples_per_second': 1.329, 'eval_steps_per_second': 0.452, 'epoch': 6.67}\n"480 ]481 },482 {483 "name": "stderr",484 "output_type": "stream",485 "text": [486 "g:\\IDEs and Modules\\Anaconda\\envs\\pytorch_gpu\\lib\\site-packages\\transformers\\modeling_utils.py:884: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers.\n",487 " warnings.warn(\n",488 " \n",489 " 60%|██████ | 600/1000 [36:52<19:39, 2.95s/it]"490 ]491 },492 {493 "name": "stdout",494 "output_type": "stream",495 "text": [496 "{'eval_loss': 0.5118691921234131, 'eval_precision': 0.8847441415590627, 'eval_recall': 0.9190263288623944, 'eval_f1': 0.9015594541910332, 'eval_accuracy': 0.8628313324616664, 'eval_runtime': 37.2452, 'eval_samples_per_second': 1.342, 'eval_steps_per_second': 0.456, 'epoch': 8.0}\n"497 ]498 },499 {500 "name": "stderr",501 "output_type": "stream",502 "text": [503 " \n",504 " 70%|███████ | 700/1000 [42:26<19:21, 3.87s/it]"505 ]506 },507 {508 "name": "stdout",509 "output_type": "stream",510 "text": [511 "{'eval_loss': 0.5196597576141357, 'eval_precision': 0.8799428299190091, 'eval_recall': 0.9175360158966717, 'eval_f1': 0.8983463035019456, 'eval_accuracy': 0.8707951979079995, 'eval_runtime': 38.9063, 'eval_samples_per_second': 1.285, 'eval_steps_per_second': 0.437, 'epoch': 9.33}\n"512 ]513 },514 {515 "name": "stderr",516 "output_type": "stream",517 "text": [518 " \n",519 " 80%|████████ | 800/1000 [49:13<12:57, 3.89s/it]"520 ]521 },522 {523 "name": "stdout",524 "output_type": "stream",525 "text": [526 "{'eval_loss': 0.5508859157562256, 'eval_precision': 0.8907603464870067, 'eval_recall': 0.9195230998509687, 'eval_f1': 0.9049132241505743, 'eval_accuracy': 0.8629501961250445, 'eval_runtime': 41.8293, 'eval_samples_per_second': 1.195, 'eval_steps_per_second': 0.406, 'epoch': 10.67}\n"527 ]528 },529 {530 "name": "stderr",531 "output_type": "stream",532 "text": [533 " \n",534 " 90%|█████████ | 900/1000 [55:10<04:35, 2.75s/it]"535 ]536 },537 {538 "name": "stdout",539 "output_type": "stream",540 "text": [541 "{'eval_loss': 0.5496693253517151, 'eval_precision': 0.8995098039215687, 'eval_recall': 0.9115747640337805, 'eval_f1': 0.9055020972119417, 'eval_accuracy': 0.8654463330559848, 'eval_runtime': 34.7336, 'eval_samples_per_second': 1.44, 'eval_steps_per_second': 0.489, 'epoch': 12.0}\n"542 ]543 },544 {545 "name": "stderr",546 "output_type": "stream",547 "text": [548 "100%|██████████| 1000/1000 [1:01:33<00:00, 3.32s/it]"549 ]550 },551 {552 "name": "stdout",553 "output_type": "stream",554 "text": [555 "{'loss': 0.123, 'learning_rate': 0.0, 'epoch': 13.33}\n"556 ]557 },558 {559 "name": "stderr",560 "output_type": "stream",561 "text": [562 " \n",563 "100%|██████████| 1000/1000 [1:02:05<00:00, 3.32s/it]"564 ]565 },566 {567 "name": "stdout",568 "output_type": "stream",569 "text": [570 "{'eval_loss': 0.5414420962333679, 'eval_precision': 0.8941005802707931, 'eval_recall': 0.9185295578738202, 'eval_f1': 0.9061504533202647, 'eval_accuracy': 0.8693688339474622, 'eval_runtime': 32.6105, 'eval_samples_per_second': 1.533, 'eval_steps_per_second': 0.521, 'epoch': 13.33}\n"571 ]572 },573 {574 "name": "stderr",575 "output_type": "stream",576 "text": [577 "100%|██████████| 1000/1000 [1:02:26<00:00, 3.75s/it]"578 ]579 },580 {581 "name": "stdout",582 "output_type": "stream",583 "text": [584 "{'train_runtime': 3746.26, 'train_samples_per_second': 0.534, 'train_steps_per_second': 0.267, 'train_loss': 0.3317529182434082, 'epoch': 13.33}\n"585 ]586 },587 {588 "name": "stderr",589 "output_type": "stream",590 "text": [591 "\n"592 ]593 },594 {595 "data": {596 "text/plain": [597 "TrainOutput(global_step=1000, training_loss=0.3317529182434082, metrics={'train_runtime': 3746.26, 'train_samples_per_second': 0.534, 'train_steps_per_second': 0.267, 'train_loss': 0.3317529182434082, 'epoch': 13.33})"598 ]599 },600 "execution_count": 13,601 "metadata": {},602 "output_type": "execute_result"603 }604 ],605 "source": [606 "trainer.train()"607 ]608 },609 {610 "cell_type": "code",611 "execution_count": 14,612 "metadata": {613 "execution": {614 "iopub.execute_input": "2023-03-03T19:24:05.222740Z",615 "iopub.status.busy": "2023-03-03T19:24:05.222132Z",616 "iopub.status.idle": "2023-03-03T19:24:09.031814Z",617 "shell.execute_reply": "2023-03-03T19:24:09.030506Z",618 "shell.execute_reply.started": "2023-03-03T19:24:05.222692Z"619 },620 "trusted": true621 },622 "outputs": [623 {624 "name": "stderr",625 "output_type": "stream",626 "text": [627 "g:\\IDEs and Modules\\Anaconda\\envs\\pytorch_gpu\\lib\\site-packages\\transformers\\modeling_utils.py:884: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers.\n",628 " warnings.warn(\n",629 "100%|██████████| 17/17 [00:50<00:00, 2.94s/it]\n"630 ]631 },632 {633 "data": {634 "text/plain": [635 "{'eval_loss': 0.5414420962333679,\n",636 " 'eval_precision': 0.8941005802707931,\n",637 " 'eval_recall': 0.9185295578738202,\n",638 " 'eval_f1': 0.9061504533202647,\n",639 " 'eval_accuracy': 0.8693688339474622,\n",640 " 'eval_runtime': 53.511,\n",641 " 'eval_samples_per_second': 0.934,\n",642 " 'eval_steps_per_second': 0.318,\n",643 " 'epoch': 13.33}"644 ]645 },646 "execution_count": 14,647 "metadata": {},648 "output_type": "execute_result"649 }650 ],651 "source": [652 "trainer.evaluate()"653 ]654 },655 {656 "cell_type": "code",657 "execution_count": 15,658 "metadata": {659 "collapsed": false660 },661 "outputs": [],662 "source": [663 "trainer.save_model(\"trained_model\")\n"664 ]665 },666 {667 "cell_type": "code",668 "execution_count": 4,669 "metadata": {},670 "outputs": [671 {672 "ename": "NameError",673 "evalue": "name 'LayoutLMv3ForTokenClassification' is not defined",674 "output_type": "error",675 "traceback": [676 "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",677 "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",678 "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m model \u001b[39m=\u001b[39m LayoutLMv3ForTokenClassification\u001b[39m.\u001b[39mfrom_pretrained(\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mG:/Form understanding in Noisy scanned documents/trained_model\u001b[39m\u001b[39m\"\u001b[39m)\n",679 "\u001b[1;31mNameError\u001b[0m: name 'LayoutLMv3ForTokenClassification' is not defined"680 ]681 },682 {683 "ename": "",684 "evalue": "",685 "output_type": "error",686 "traceback": [687 "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."688 ]689 }690 ],691 "source": [692 "model = LayoutLMv3ForTokenClassification.from_pretrained(r\"G:/Form understanding in Noisy scanned documents/trained_model\")"693 ]694 },695 {696 "cell_type": "code",697 "execution_count": null,698 "metadata": {699 "trusted": true700 },701 "outputs": [],702 "source": [703 "device = torch.device(\"cuda\")\n",704 "model.cuda()"705 ]706 },707 {708 "cell_type": "code",709 "execution_count": null,710 "metadata": {711 "execution": {712 "iopub.execute_input": "2023-03-03T19:43:35.233219Z",713 "iopub.status.busy": "2023-03-03T19:43:35.232609Z",714 "iopub.status.idle": "2023-03-03T19:43:35.335952Z",715 "shell.execute_reply": "2023-03-03T19:43:35.334111Z",716 "shell.execute_reply.started": "2023-03-03T19:43:35.233171Z"717 },718 "trusted": true719 },720 "outputs": [],721 "source": [722 "example = dataset[\"test\"][2]\n",723 "print(example.keys())\n",724 "\n",725 "image = example[\"image\"]\n",726 "words = example[\"tokens\"]\n",727 "boxes = example[\"bboxes\"]\n",728 "word_labels = example[\"ner_tags\"]\n",729 "\n",730 "encoding = tokenizer(image, words, boxes=boxes, word_labels=word_labels, return_tensors=\"pt\")\n",731 "encoding = encoding.to('cuda')\n",732 "for k,v in encoding.items():\n",733 " print(k,v.shape)\n",734 "\n",735 "with torch.no_grad():\n",736 " outputs = model.to('cuda')(**encoding)\n",737 "\n",738 "logits = outputs.logits\n",739 "logits.shape"740 ]741 },742 {743 "cell_type": "code",744 "execution_count": null,745 "metadata": {746 "execution": {747 "iopub.execute_input": "2023-03-03T19:43:41.414753Z",748 "iopub.status.busy": "2023-03-03T19:43:41.413702Z",749 "iopub.status.idle": "2023-03-03T19:43:41.427778Z",750 "shell.execute_reply": "2023-03-03T19:43:41.424566Z",751 "shell.execute_reply.started": "2023-03-03T19:43:41.414704Z"752 },753 "trusted": true754 },755 "outputs": [],756 "source": [757 "predictions = logits.argmax(-1).squeeze().tolist()\n",758 "#labels = encoding.labels.squeeze().tolist()\n"759 ]760 },761 {762 "attachments": {},763 "cell_type": "markdown",764 "metadata": {},765 "source": []766 },767 {768 "cell_type": "code",769 "execution_count": null,770 "metadata": {771 "execution": {772 "iopub.execute_input": "2023-03-03T20:05:26.719238Z",773 "iopub.status.busy": "2023-03-03T20:05:26.718647Z",774 "iopub.status.idle": "2023-03-03T20:05:26.731470Z",775 "shell.execute_reply": "2023-03-03T20:05:26.730335Z",776 "shell.execute_reply.started": "2023-03-03T20:05:26.719199Z"777 },778 "trusted": true779 },780 "outputs": [],781 "source": [782 "def unnormalize_box(bbox, width, height):\n",783 " return [\n",784 " width * (bbox[0] / 1000),\n",785 " height * (bbox[1] / 1000),\n",786 " width * (bbox[2] / 1000),\n",787 " height * (bbox[3] / 1000),\n",788 " ]\n",789 "\n",790 "token_boxes = encoding.bbox.squeeze().tolist()\n",791 "width, height = image.size\n",792 "\n",793 "true_predictions = [model.config.id2label[pred] for pred, label in zip(predictions, labels) if label != - 100]\n",794 "true_labels = [model.config.id2label[label] for prediction, label in zip(predictions, labels) if label != -100]\n",795 "true_boxes = [unnormalize_box(box, width, height) for box, label in zip(token_boxes, labels) if label != -100]"796 ]797 },798 {799 "cell_type": "code",800 "execution_count": null,801 "metadata": {802 "execution": {803 "iopub.execute_input": "2023-03-03T20:05:31.060497Z",804 "iopub.status.busy": "2023-03-03T20:05:31.060140Z",805 "iopub.status.idle": "2023-03-03T20:05:31.161089Z",806 "shell.execute_reply": "2023-03-03T20:05:31.158110Z",807 "shell.execute_reply.started": "2023-03-03T20:05:31.060465Z"808 },809 "trusted": true810 },811 "outputs": [],812 "source": [813 "from PIL import ImageDraw, ImageFont\n",814 "\n",815 "draw = ImageDraw.Draw(image)\n",816 "\n",817 "font = ImageFont.load_default()\n",818 "\n",819 "def iob_to_label(label):\n",820 " label = label[2:]\n",821 " if not label:\n",822 " return 'other'\n",823 " return label\n",824 "\n",825 "label2color = {'question':'blue', 'answer':'green', 'header':'orange', 'other':'violet'}\n",826 "\n",827 "for prediction, box in zip(true_predictions, true_boxes):\n",828 " predicted_label = iob_to_label(prediction).lower()\n",829 " draw.rectangle(box, outline=label2color[predicted_label])\n",830 " draw.text((box[0] + 10, box[1] - 10), text=predicted_label, fill=label2color[predicted_label], font=font)\n",831 "\n",832 "image"833 ]834 }835 ],836 "metadata": {837 "kernelspec": {838 "display_name": "pytorch_gpu",839 "language": "python",840 "name": "python3"841 },842 "language_info": {843 "codemirror_mode": {844 "name": "ipython",845 "version": 3846 },847 "file_extension": ".py",848 "mimetype": "text/x-python",849 "name": "python",850 "nbconvert_exporter": "python",851 "pygments_lexer": "ipython3",852 "version": "3.10.11"853 },854 "polyglot_notebook": {855 "kernelInfo": {856 "defaultKernelName": "csharp",857 "items": [858 {859 "aliases": [],860 "name": "csharp"861 }862 ]863 }864 }865 },866 "nbformat": 4,867 "nbformat_minor": 4868}869