Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 201592,7 "name": "Neil Mehta",8 "username": "neilmehta87",9 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png",10 "created_at": "2020-06-10T02:32:02.065Z",11 "cooked": "<p>Hi,</p>\n<p>I am trying to write a simple NN example in libtorch and I was able to successfully run this code on a CPU. However, after changing torch::Device to kCUDA, I was able to run the code on a GPU but it throws a seg fault error after finishing the run. Could someone please help me?<br>\nI am attaching my code below</p>\n<pre><code class=\"lang-auto\">#include<torch/torch.h>\n#include<iostream>\n#include<cmath>\n#include<cstdio>\n\n//The batch size for training\nconst int64_t N = 64;\n//The input dimension\nconst int64_t D_in = 1000;\n//The hidden dimension\nconst int64_t H = 100;\n//The output dimension\nconst int64_t D_out = 10;\n//Total number of steps\nconst int64_t tstep = 1000;\n\nusing namespace torch;\nusing namespace std;\n\nstruct TwoLayerNetImpl : nn::Module{\n TwoLayerNetImpl(int D_in, int H, int D_out)\n : linear1(nn::LinearOptions(D_in, H).bias(false)),\n\tlinear2(nn::LinearOptions(H, D_out).bias(false))\n {\n register_module(\"linear1\", linear1);\n register_module(\"linear2\", linear2);\n }\n\n torch::Tensor forward(torch::Tensor x)\n {\n x = torch::clamp_min(linear1(x),0);\n x = linear2(x);\n return x;\n }\n nn::Linear linear1, linear2;\n};\nTORCH_MODULE(TwoLayerNet);\n\nint main(int argc, char* argv[])\n{\n torch::manual_seed(1);\n torch::Device device(torch::kCUDA); //change to kCPU to run on host\n if (torch::cuda::is_available())\n {\n std::cout << \"CUDA is available! Training on GPU\" << std::endl;\n }\n\n torch::Tensor X = torch::randn({N, D_in}).to(device);\n torch::Tensor Y = torch::randn({N, D_out}).to(device);\n \n TwoLayerNet net(D_in,H,D_out);\n net->to(device);\n \n torch::optim::Adam optimizer(net->parameters(), torch::optim::AdamOptions(1e-4));\n torch::nn::MSELoss criterion((torch::nn::MSELossOptions(torch::kSum)));\n \n for(int64_t ts=0;ts<=tstep;++ts)\n {\n torch::Tensor Y_pred = net->forward(X);\n torch::Tensor loss = criterion(Y_pred, Y);\n if(ts % 100 == 0)\n {\n printf(\"\\r[%4ld/%4ld] | D_loss: %e \\n\", ts,tstep,loss.item<float>());\n }\n optimizer.zero_grad();\n loss.backward();\n optimizer.step();\n }\n std::cout<<\"Training complete!\"<<std::endl;\n return 0;\n}\n</code></pre>\n<p>Thank you!</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 5,15 "updated_at": "2020-06-10T02:33:22.709Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 294,20 "reads": 20,21 "readers_count": 19,22 "score": 1474.0,23 "yours": false,24 "topic_id": 84832,25 "topic_slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",26 "display_username": "Neil Mehta",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 32062,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu/84832/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 201932,64 "name": "Xinyu Li Pytorch Dev",65 "username": "glaringlee",66 "avatar_template": "/letter_avatar_proxy/v4/letter/g/9de053/{size}.png",67 "created_at": "2020-06-10T20:20:14.800Z",68 "cooked": "<p>Can you provide which libtorch version (including cuda version) you are using?<br>\nWhat’s your cuda version?<br>\nCan you paste your error msg?</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 5,72 "updated_at": "2020-06-10T20:20:14.800Z",73 "reply_count": 0,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 2,77 "reads": 15,78 "readers_count": 14,79 "score": 13.0,80 "yours": false,81 "topic_id": 84832,82 "topic_slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",83 "display_username": "Xinyu Li Pytorch Dev",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": false,102 "admin": false,103 "staff": false,104 "user_id": 26938,105 "hidden": false,106 "trust_level": 2,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu/84832/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 201935,120 "name": "Neil Mehta",121 "username": "neilmehta87",122 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png",123 "created_at": "2020-06-10T20:43:43.880Z",124 "cooked": "<p>Hi,</p>\n<p>I am using pytorch/v1.5.0-gpu. I downloaded the stable release of libtorch.<br>\nI was using cuda 10.1.243 yesterday and it was giving me a segmentation fault error.<br>\nHowever, after switching to cuda 10.2.89 and recompiling my code, I no longer get the error.<br>\nI apologize for the false alarm.</p>\n<p>Thank you</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 5,128 "updated_at": "2020-06-10T20:43:43.880Z",129 "reply_count": 0,130 "reply_to_post_number": null,131 "quote_count": 0,132 "incoming_link_count": 3,133 "reads": 14,134 "readers_count": 13,135 "score": 17.8,136 "yours": false,137 "topic_id": 84832,138 "topic_slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",139 "display_username": "Neil Mehta",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 1,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": null,155 "bookmarked": false,156 "actions_summary": [],157 "moderator": false,158 "admin": false,159 "staff": false,160 "user_id": 32062,161 "hidden": false,162 "trust_level": 1,163 "deleted_at": null,164 "user_deleted": false,165 "edit_reason": null,166 "can_view_edit_history": true,167 "wiki": false,168 "post_url": "/t/code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu/84832/3",169 "can_accept_answer": false,170 "can_unaccept_answer": false,171 "accepted_answer": false,172 "topic_accepted_answer": null173 },174 {175 "id": 201998,176 "name": "Xinyu Li Pytorch Dev",177 "username": "glaringlee",178 "avatar_template": "/letter_avatar_proxy/v4/letter/g/9de053/{size}.png",179 "created_at": "2020-06-11T03:12:04.806Z",180 "cooked": "<p><a class=\"mention\" href=\"/u/neilmehta87\">@neilmehta87</a></p>\n<p>Glad to here that. One possible reason is that the libtorch you downloaded is for cuda 10.2.<br>\nWe have different libtorch binary for different version of cuda.<br>\nYou can find them here: <a href=\"https://pytorch.org/\" rel=\"nofollow noopener\">https://pytorch.org/</a></p>",181 "post_number": 4,182 "post_type": 1,183 "posts_count": 5,184 "updated_at": "2020-06-11T03:12:04.806Z",185 "reply_count": 1,186 "reply_to_post_number": null,187 "quote_count": 0,188 "incoming_link_count": 0,189 "reads": 12,190 "readers_count": 11,191 "score": 7.4,192 "yours": false,193 "topic_id": 84832,194 "topic_slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",195 "display_username": "Xinyu Li Pytorch Dev",196 "primary_group_name": null,197 "flair_name": null,198 "flair_url": null,199 "flair_bg_color": null,200 "flair_color": null,201 "flair_group_id": null,202 "badges_granted": [],203 "version": 1,204 "can_edit": false,205 "can_delete": false,206 "can_recover": false,207 "can_see_hidden_post": false,208 "can_wiki": false,209 "link_counts": [210 {211 "url": "https://pytorch.org/",212 "internal": false,213 "reflection": false,214 "title": "PyTorch",215 "clicks": 16216 }217 ],218 "read": true,219 "user_title": "",220 "bookmarked": false,221 "actions_summary": [],222 "moderator": false,223 "admin": false,224 "staff": false,225 "user_id": 26938,226 "hidden": false,227 "trust_level": 2,228 "deleted_at": null,229 "user_deleted": false,230 "edit_reason": null,231 "can_view_edit_history": true,232 "wiki": false,233 "post_url": "/t/code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu/84832/4",234 "can_accept_answer": false,235 "can_unaccept_answer": false,236 "accepted_answer": false,237 "topic_accepted_answer": null238 },239 {240 "id": 202021,241 "name": "Neil Mehta",242 "username": "neilmehta87",243 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png",244 "created_at": "2020-06-11T05:54:48.770Z",245 "cooked": "<p><a class=\"mention\" href=\"/u/glaringlee\">@glaringlee</a> Thank you</p>",246 "post_number": 5,247 "post_type": 1,248 "posts_count": 5,249 "updated_at": "2020-06-11T05:54:48.770Z",250 "reply_count": 0,251 "reply_to_post_number": 4,252 "quote_count": 0,253 "incoming_link_count": 1,254 "reads": 11,255 "readers_count": 10,256 "score": 7.2,257 "yours": false,258 "topic_id": 84832,259 "topic_slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",260 "display_username": "Neil Mehta",261 "primary_group_name": null,262 "flair_name": null,263 "flair_url": null,264 "flair_bg_color": null,265 "flair_color": null,266 "flair_group_id": null,267 "badges_granted": [],268 "version": 1,269 "can_edit": false,270 "can_delete": false,271 "can_recover": false,272 "can_see_hidden_post": false,273 "can_wiki": false,274 "read": true,275 "user_title": null,276 "reply_to_user": {277 "id": 26938,278 "username": "glaringlee",279 "name": "Xinyu Li Pytorch Dev",280 "avatar_template": "/letter_avatar_proxy/v4/letter/g/9de053/{size}.png"281 },282 "bookmarked": false,283 "actions_summary": [],284 "moderator": false,285 "admin": false,286 "staff": false,287 "user_id": 32062,288 "hidden": false,289 "trust_level": 1,290 "deleted_at": null,291 "user_deleted": false,292 "edit_reason": null,293 "can_view_edit_history": true,294 "wiki": false,295 "post_url": "/t/code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu/84832/5",296 "can_accept_answer": false,297 "can_unaccept_answer": false,298 "accepted_answer": false,299 "topic_accepted_answer": null300 }301 ],302 "stream": [303 201592,304 201932,305 201935,306 201998,307 202021308 ]309 },310 "timeline_lookup": [311 [312 1,313 1964314 ],315 [316 2,317 1963318 ]319 ],320 "suggested_topics": [321 {322 "fancy_title": "Libtorch, Cannot find NVTX3, find old NVTX instead",323 "id": 213641,324 "title": "Libtorch, Cannot find NVTX3, find old NVTX instead",325 "slug": "libtorch-cannot-find-nvtx3-find-old-nvtx-instead",326 "posts_count": 3,327 "reply_count": 1,328 "highest_post_number": 3,329 "image_url": null,330 "created_at": "2024-11-30T18:43:13.619Z",331 "last_posted_at": "2025-07-06T09:49:09.711Z",332 "bumped": true,333 "bumped_at": "2025-07-06T09:49:09.711Z",334 "archetype": "regular",335 "unseen": false,336 "pinned": false,337 "unpinned": null,338 "visible": true,339 "closed": false,340 "archived": false,341 "bookmarked": null,342 "liked": null,343 "tags_descriptions": {},344 "like_count": 3,345 "views": 2384,346 "category_id": 11,347 "featured_link": null,348 "has_accepted_answer": true,349 "posters": [350 {351 "extras": null,352 "description": "Original Poster, Accepted Answer",353 "user": {354 "id": 65591,355 "username": "weitaoliu",356 "name": "",357 "avatar_template": "/letter_avatar_proxy/v4/letter/w/35a633/{size}.png",358 "trust_level": 1359 }360 },361 {362 "extras": "latest",363 "description": "Most Recent Poster",364 "user": {365 "id": 35595,366 "username": "5had3z",367 "name": "5had3z",368 "avatar_template": "/user_avatar/discuss.pytorch.org/5had3z/{size}/28066_2.png",369 "trust_level": 1370 }371 }372 ]373 },374 {375 "fancy_title": "Libtorch low latency problem",376 "id": 216172,377 "title": "Libtorch low latency problem",378 "slug": "libtorch-low-latency-problem",379 "posts_count": 4,380 "reply_count": 1,381 "highest_post_number": 4,382 "image_url": null,383 "created_at": "2025-02-03T12:39:55.423Z",384 "last_posted_at": "2025-02-10T15:52:08.960Z",385 "bumped": true,386 "bumped_at": "2025-02-10T15:52:08.960Z",387 "archetype": "regular",388 "unseen": false,389 "pinned": false,390 "unpinned": null,391 "visible": true,392 "closed": false,393 "archived": false,394 "bookmarked": null,395 "liked": null,396 "tags_descriptions": {},397 "like_count": 0,398 "views": 225,399 "category_id": 11,400 "featured_link": null,401 "has_accepted_answer": false,402 "posters": [403 {404 "extras": null,405 "description": "Original Poster",406 "user": {407 "id": 82454,408 "username": "Iwein",409 "name": "Dr. ir. ing. I.Vranckx",410 "avatar_template": "/user_avatar/discuss.pytorch.org/iwein/{size}/75439_2.png",411 "trust_level": 1412 }413 },414 {415 "extras": "latest",416 "description": "Most Recent Poster",417 "user": {418 "id": 1,419 "username": "smth",420 "name": "",421 "avatar_template": "/user_avatar/discuss.pytorch.org/smth/{size}/13_2.png",422 "admin": true,423 "moderator": true,424 "trust_level": 2425 }426 }427 ]428 },429 {430 "fancy_title": "Libtorch multi GPU training",431 "id": 216479,432 "title": "Libtorch multi GPU training",433 "slug": "libtorch-multi-gpu-training",434 "posts_count": 2,435 "reply_count": 0,436 "highest_post_number": 2,437 "image_url": null,438 "created_at": "2025-02-10T15:05:26.160Z",439 "last_posted_at": "2025-03-20T14:22:30.431Z",440 "bumped": true,441 "bumped_at": "2025-03-20T14:22:30.431Z",442 "archetype": "regular",443 "unseen": false,444 "pinned": false,445 "unpinned": null,446 "visible": true,447 "closed": false,448 "archived": false,449 "bookmarked": null,450 "liked": null,451 "tags_descriptions": {},452 "like_count": 0,453 "views": 106,454 "category_id": 11,455 "featured_link": null,456 "has_accepted_answer": false,457 "posters": [458 {459 "extras": "latest single",460 "description": "Original Poster, Most Recent Poster",461 "user": {462 "id": 81205,463 "username": "kevin2004",464 "name": "",465 "avatar_template": "/user_avatar/discuss.pytorch.org/kevin2004/{size}/74269_2.png",466 "trust_level": 0467 }468 }469 ]470 },471 {472 "fancy_title": "Generate a sub-2D tensor by reference",473 "id": 214275,474 "title": "Generate a sub-2D tensor by reference",475 "slug": "generate-a-sub-2d-tensor-by-reference",476 "posts_count": 2,477 "reply_count": 0,478 "highest_post_number": 2,479 "image_url": null,480 "created_at": "2024-12-16T15:28:23.598Z",481 "last_posted_at": "2024-12-16T17:04:40.717Z",482 "bumped": true,483 "bumped_at": "2024-12-16T17:04:40.717Z",484 "archetype": "regular",485 "unseen": false,486 "pinned": false,487 "unpinned": null,488 "visible": true,489 "closed": false,490 "archived": false,491 "bookmarked": null,492 "liked": null,493 "tags_descriptions": {},494 "like_count": 1,495 "views": 31,496 "category_id": 11,497 "featured_link": null,498 "has_accepted_answer": false,499 "posters": [500 {501 "extras": null,502 "description": "Original Poster",503 "user": {504 "id": 65591,505 "username": "weitaoliu",506 "name": "",507 "avatar_template": "/letter_avatar_proxy/v4/letter/w/35a633/{size}.png",508 "trust_level": 1509 }510 },511 {512 "extras": "latest",513 "description": "Most Recent Poster",514 "user": {515 "id": 3534,516 "username": "ptrblck",517 "name": "",518 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",519 "admin": true,520 "moderator": true,521 "trust_level": 2522 }523 }524 ]525 },526 {527 "fancy_title": "ANN implementation using c++ for a text classification model",528 "id": 219040,529 "title": "ANN implementation using c++ for a text classification model",530 "slug": "ann-implementation-using-c-for-a-text-classification-model",531 "posts_count": 1,532 "reply_count": 0,533 "highest_post_number": 1,534 "image_url": null,535 "created_at": "2025-04-14T02:27:58.483Z",536 "last_posted_at": "2025-04-14T02:27:58.520Z",537 "bumped": true,538 "bumped_at": "2025-04-14T02:27:58.520Z",539 "archetype": "regular",540 "unseen": false,541 "pinned": false,542 "unpinned": null,543 "visible": true,544 "closed": false,545 "archived": false,546 "bookmarked": null,547 "liked": null,548 "tags_descriptions": {},549 "like_count": 0,550 "views": 29,551 "category_id": 11,552 "featured_link": null,553 "has_accepted_answer": false,554 "posters": [555 {556 "extras": "latest single",557 "description": "Original Poster, Most Recent Poster",558 "user": {559 "id": 83566,560 "username": "Hits",561 "name": "",562 "avatar_template": "/letter_avatar_proxy/v4/letter/h/779978/{size}.png",563 "trust_level": 1564 }565 }566 ]567 }568 ],569 "tags_descriptions": {},570 "fancy_title": "Code runs fine on CPU and GPU but gives seg fault at the end onGPU",571 "id": 84832,572 "title": "Code runs fine on CPU and GPU but gives seg fault at the end onGPU",573 "posts_count": 5,574 "created_at": "2020-06-10T02:32:02.015Z",575 "views": 1068,576 "reply_count": 1,577 "like_count": 0,578 "last_posted_at": "2020-06-11T05:54:48.770Z",579 "visible": true,580 "closed": false,581 "archived": false,582 "has_summary": false,583 "archetype": "regular",584 "slug": "code-runs-fine-on-cpu-and-gpu-but-gives-seg-fault-at-the-end-ongpu",585 "category_id": 11,586 "word_count": 425,587 "deleted_at": null,588 "user_id": 32062,589 "featured_link": null,590 "pinned_globally": false,591 "pinned_at": null,592 "pinned_until": null,593 "image_url": null,594 "slow_mode_seconds": 0,595 "draft": null,596 "draft_key": "topic_84832",597 "draft_sequence": null,598 "unpinned": null,599 "pinned": false,600 "current_post_number": 1,601 "highest_post_number": 5,602 "deleted_by": null,603 "actions_summary": [604 {605 "id": 4,606 "count": 0,607 "hidden": false,608 "can_act": false609 },610 {611 "id": 8,612 "count": 0,613 "hidden": false,614 "can_act": false615 },616 {617 "id": 10,618 "count": 0,619 "hidden": false,620 "can_act": false621 },622 {623 "id": 7,624 "count": 0,625 "hidden": false,626 "can_act": false627 }628 ],629 "chunk_size": 20,630 "bookmarked": false,631 "topic_timer": null,632 "message_bus_last_id": 0,633 "participant_count": 2,634 "show_read_indicator": false,635 "thumbnails": null,636 "slow_mode_enabled_until": null,637 "can_vote": false,638 "vote_count": 0,639 "user_voted": false,640 "discourse_zendesk_plugin_zendesk_id": null,641 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",642 "details": {643 "can_edit": false,644 "notification_level": 1,645 "participants": [646 {647 "id": 32062,648 "username": "neilmehta87",649 "name": "Neil Mehta",650 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png",651 "post_count": 3,652 "primary_group_name": null,653 "flair_name": null,654 "flair_url": null,655 "flair_color": null,656 "flair_bg_color": null,657 "flair_group_id": null,658 "trust_level": 1659 },660 {661 "id": 26938,662 "username": "glaringlee",663 "name": "Xinyu Li Pytorch Dev",664 "avatar_template": "/letter_avatar_proxy/v4/letter/g/9de053/{size}.png",665 "post_count": 2,666 "primary_group_name": null,667 "flair_name": null,668 "flair_url": null,669 "flair_color": null,670 "flair_bg_color": null,671 "flair_group_id": null,672 "trust_level": 2673 }674 ],675 "created_by": {676 "id": 32062,677 "username": "neilmehta87",678 "name": "Neil Mehta",679 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png"680 },681 "last_poster": {682 "id": 32062,683 "username": "neilmehta87",684 "name": "Neil Mehta",685 "avatar_template": "/user_avatar/discuss.pytorch.org/neilmehta87/{size}/25835_2.png"686 },687 "links": [688 {689 "url": "https://pytorch.org/",690 "title": "PyTorch",691 "internal": false,692 "attachment": false,693 "reflection": false,694 "clicks": 16,695 "user_id": 26938,696 "domain": "pytorch.org",697 "root_domain": "pytorch.org"698 }699 ]700 },701 "bookmarks": []702 },703 {704 "post_stream": {705 "posts": [706 {707 "id": 198104,708 "name": "Ali Tahir",709 "username": "alitahir",710 "avatar_template": "/user_avatar/discuss.pytorch.org/alitahir/{size}/11828_2.png",711 "created_at": "2020-05-29T17:26:18.603Z",712 "cooked": "<p>I’m using transfer learning with k-fold cross validation. k=9 and k folds are being generated using sklearn. The problem i’m facing is that after evaluation of first fold performance decreases drastically. In first fold i get 54% validation accuracy and after that even the training accuracy doesn’t exceed 4% for all the remaining folds.</p>\n<p>First guess was that it is probably because of data difference in all folds. So i have run the model multiple times but this behavior still exists. It doesn’t seem to be because of data as new run generates shuffled new data for all folds. I’m completely out of ideas why its happening. Any help is appreciated. Thank You.</p>\n<blockquote>\n<p>Blockquote<br>\ndef loadModel():<br>\nmodel = sphere20a(classnum=1000).to(device)<br>\nmodel_state = model.state_dict()<br>\nfor name, param in model.named_parameters():<br>\nif (name[:4] == ‘conv’ or name[:4] == ‘relu’) and name[4] is not ‘4’:<br>\nparam.requires_grad = False</p>\n</blockquote>\n<pre><code>pretrained_state = torch.load(\"/kaggle/input/kinmodel/sphere20a_20171020.pth\")\npretrained_state = {k:v for k,v in pretrained_state.items() if k in model_state and v.size() == model_state[k].size()}\n\nmodel_state.update(pretrained_state)\nmodel.load_state_dict(model_state)\nreturn model\n</code></pre>\n<p>training loop.</p>\n<p>for train_index, val_index in kf.split(data):</p>\n<pre><code>n_epochs=10\nlearn_rate = 0.01\ncounter = counter + 1\nmodel = loadModel() #load new model everytime and discard old one\n</code></pre>\n<p>I am deleting the old model after every fold completes. using del model</p>",713 "post_number": 1,714 "post_type": 1,715 "posts_count": 4,716 "updated_at": "2020-05-29T17:27:34.532Z",717 "reply_count": 0,718 "reply_to_post_number": null,719 "quote_count": 0,720 "incoming_link_count": 43,721 "reads": 13,722 "readers_count": 12,723 "score": 217.6,724 "yours": false,725 "topic_id": 83401,726 "topic_slug": "kfold-cross-validation-provides-extremely-unexpected-results",727 "display_username": "Ali Tahir",728 "primary_group_name": null,729 "flair_name": null,730 "flair_url": null,731 "flair_bg_color": null,732 "flair_color": null,733 "flair_group_id": null,734 "badges_granted": [],735 "version": 1,736 "can_edit": false,737 "can_delete": false,738 "can_recover": false,739 "can_see_hidden_post": false,740 "can_wiki": false,741 "read": true,742 "user_title": null,743 "bookmarked": false,744 "actions_summary": [],745 "moderator": false,746 "admin": false,747 "staff": false,748 "user_id": 32366,749 "hidden": false,750 "trust_level": 0,751 "deleted_at": null,752 "user_deleted": false,753 "edit_reason": null,754 "can_view_edit_history": true,755 "wiki": false,756 "post_url": "/t/kfold-cross-validation-provides-extremely-unexpected-results/83401/1",757 "can_accept_answer": false,758 "can_unaccept_answer": false,759 "accepted_answer": false,760 "topic_accepted_answer": null,761 "can_vote": false762 },763 {764 "id": 198106,765 "name": "Ali Tahir",766 "username": "alitahir",767 "avatar_template": "/user_avatar/discuss.pytorch.org/alitahir/{size}/11828_2.png",768 "created_at": "2020-05-29T17:29:46.499Z",769 "cooked": "<p>If somebody is up for the challenge of finding the problem i can share the complete notebook.</p>",770 "post_number": 2,771 "post_type": 1,772 "posts_count": 4,773 "updated_at": "2020-05-29T17:29:46.499Z",774 "reply_count": 1,775 "reply_to_post_number": null,776 "quote_count": 0,777 "incoming_link_count": 1,778 "reads": 12,779 "readers_count": 11,780 "score": 12.4,781 "yours": false,782 "topic_id": 83401,783 "topic_slug": "kfold-cross-validation-provides-extremely-unexpected-results",784 "display_username": "Ali Tahir",785 "primary_group_name": null,786 "flair_name": null,787 "flair_url": null,788 "flair_bg_color": null,789 "flair_color": null,790 "flair_group_id": null,791 "badges_granted": [],792 "version": 1,793 "can_edit": false,794 "can_delete": false,795 "can_recover": false,796 "can_see_hidden_post": false,797 "can_wiki": false,798 "read": true,799 "user_title": null,800 "bookmarked": false,801 "actions_summary": [],802 "moderator": false,803 "admin": false,804 "staff": false,805 "user_id": 32366,806 "hidden": false,807 "trust_level": 0,808 "deleted_at": null,809 "user_deleted": false,810 "edit_reason": null,811 "can_view_edit_history": true,812 "wiki": false,813 "post_url": "/t/kfold-cross-validation-provides-extremely-unexpected-results/83401/2",814 "can_accept_answer": false,815 "can_unaccept_answer": false,816 "accepted_answer": false,817 "topic_accepted_answer": null818 },819 {820 "id": 198272,821 "name": "",822 "username": "ptrblck",823 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",824 "created_at": "2020-05-30T12:54:20.504Z",825 "cooked": "<p>Could you post the complete code, so that I could have a look?<br>\nI’m always up for a challenge! <img src=\"https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9\" title=\":wink:\" class=\"emoji\" alt=\":wink:\"></p>",826 "post_number": 3,827 "post_type": 1,828 "posts_count": 4,829 "updated_at": "2020-05-30T12:54:20.504Z",830 "reply_count": 1,831 "reply_to_post_number": 2,832 "quote_count": 0,833 "incoming_link_count": 1,834 "reads": 11,835 "readers_count": 10,836 "score": 12.2,837 "yours": false,838 "topic_id": 83401,839 "topic_slug": "kfold-cross-validation-provides-extremely-unexpected-results",840 "display_username": "",841 "primary_group_name": null,842 "flair_name": null,843 "flair_url": null,844 "flair_bg_color": null,845 "flair_color": null,846 "flair_group_id": null,847 "badges_granted": [],848 "version": 1,849 "can_edit": false,850 "can_delete": false,851 "can_recover": false,852 "can_see_hidden_post": false,853 "can_wiki": false,854 "read": true,855 "user_title": "",856 "reply_to_user": {857 "id": 32366,858 "username": "alitahir",859 "name": "Ali Tahir",860 "avatar_template": "/user_avatar/discuss.pytorch.org/alitahir/{size}/11828_2.png"861 },862 "bookmarked": false,863 "actions_summary": [],864 "moderator": true,865 "admin": true,866 "staff": true,867 "user_id": 3534,868 "hidden": false,869 "trust_level": 2,870 "deleted_at": null,871 "user_deleted": false,872 "edit_reason": null,873 "can_view_edit_history": true,874 "wiki": false,875 "post_url": "/t/kfold-cross-validation-provides-extremely-unexpected-results/83401/3",876 "can_accept_answer": false,877 "can_unaccept_answer": false,878 "accepted_answer": false,879 "topic_accepted_answer": null880 },881 {882 "id": 202017,883 "name": "Ali Tahir",884 "username": "alitahir",885 "avatar_template": "/user_avatar/discuss.pytorch.org/alitahir/{size}/11828_2.png",886 "created_at": "2020-06-11T05:08:05.424Z",887 "cooked": "<p>Sorry for replying late. I got caught up in some stuff. I have made the notebook public. Here is the link: <a href=\"https://www.kaggle.com/ali14tahir/sphereface-model\" rel=\"nofollow noopener\">https://www.kaggle.com/ali14tahir/sphereface-model</a></p>\n<p>I can also share it with you kaggle if you like. I haven’t changed anything since then as I don’t understand what’s going on anymore</p>",888 "post_number": 4,889 "post_type": 1,890 "posts_count": 4,891 "updated_at": "2020-06-11T05:08:05.424Z",892 "reply_count": 0,893 "reply_to_post_number": 3,894 "quote_count": 0,895 "incoming_link_count": 0,896 "reads": 6,897 "readers_count": 5,898 "score": 1.2,899 "yours": false,900 "topic_id": 83401,901 "topic_slug": "kfold-cross-validation-provides-extremely-unexpected-results",902 "display_username": "Ali Tahir",903 "primary_group_name": null,904 "flair_name": null,905 "flair_url": null,906 "flair_bg_color": null,907 "flair_color": null,908 "flair_group_id": null,909 "badges_granted": [],910 "version": 1,911 "can_edit": false,912 "can_delete": false,913 "can_recover": false,914 "can_see_hidden_post": false,915 "can_wiki": false,916 "link_counts": [917 {918 "url": "https://www.kaggle.com/ali14tahir/sphereface-model",919 "internal": false,920 "reflection": false,921 "title": "SphereFace Model | Kaggle",922 "clicks": 2923 }924 ],925 "read": true,926 "user_title": null,927 "reply_to_user": {928 "id": 3534,929 "username": "ptrblck",930 "name": "",931 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"932 },933 "bookmarked": false,934 "actions_summary": [],935 "moderator": false,936 "admin": false,937 "staff": false,938 "user_id": 32366,939 "hidden": false,940 "trust_level": 0,941 "deleted_at": null,942 "user_deleted": false,943 "edit_reason": null,944 "can_view_edit_history": true,945 "wiki": false,946 "post_url": "/t/kfold-cross-validation-provides-extremely-unexpected-results/83401/4",947 "can_accept_answer": false,948 "can_unaccept_answer": false,949 "accepted_answer": false,950 "topic_accepted_answer": null951 }952 ],953 "stream": [954 198104,955 198106,956 198272,957 202017958 ]959 },960 "timeline_lookup": [961 [962 1,963 1975964 ],965 [966 3,967 1974968 ],969 [970 4,971 1963972 ]973 ],974 "suggested_topics": [975 {976 "fancy_title": "No gradient found for a parameter in custom Linear class",977 "id": 212455,978 "title": "No gradient found for a parameter in custom Linear class",979 "slug": "no-gradient-found-for-a-parameter-in-custom-linear-class",980 "posts_count": 2,981 "reply_count": 0,982 "highest_post_number": 2,983 "image_url": null,984 "created_at": "2024-11-02T19:37:13.384Z",985 "last_posted_at": "2024-11-04T14:49:26.100Z",986 "bumped": true,987 "bumped_at": "2024-11-04T14:49:26.100Z",988 "archetype": "regular",989 "unseen": false,990 "pinned": false,991 "unpinned": null,992 "visible": true,993 "closed": false,994 "archived": false,995 "bookmarked": null,996 "liked": null,997 "tags_descriptions": {},998 "like_count": 0,999 "views": 42,1000 "category_id": 5,1001 "featured_link": null,1002 "has_accepted_answer": false,1003 "posters": [1004 {1005 "extras": null,1006 "description": "Original Poster",1007 "user": {1008 "id": 80655,1009 "username": "Arkapravo_Ghosh",1010 "name": "Arkapravo Ghosh",1011 "avatar_template": "/user_avatar/discuss.pytorch.org/arkapravo_ghosh/{size}/73737_2.png",1012 "trust_level": 11013 }1014 },1015 {1016 "extras": "latest",1017 "description": "Most Recent Poster",1018 "user": {1019 "id": 3534,1020 "username": "ptrblck",1021 "name": "",1022 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1023 "admin": true,1024 "moderator": true,1025 "trust_level": 21026 }1027 }1028 ]1029 },1030 {1031 "fancy_title": "Yolo Neural Layer",1032 "id": 212691,1033 "title": "Yolo Neural Layer",1034 "slug": "yolo-neural-layer",1035 "posts_count": 1,1036 "reply_count": 0,1037 "highest_post_number": 1,1038 "image_url": null,1039 "created_at": "2024-11-08T08:05:48.745Z",1040 "last_posted_at": "2024-11-08T08:05:48.792Z",1041 "bumped": true,1042 "bumped_at": "2024-11-08T08:05:48.792Z",1043 "archetype": "regular",1044 "unseen": false,1045 "pinned": false,1046 "unpinned": null,1047 "visible": true,1048 "closed": false,1049 "archived": false,1050 "bookmarked": null,1051 "liked": null,1052 "tags_descriptions": {},1053 "like_count": 0,1054 "views": 114,1055 "category_id": 5,1056 "featured_link": null,1057 "has_accepted_answer": false,1058 "posters": [1059 {1060 "extras": "latest single",1061 "description": "Original Poster, Most Recent Poster",1062 "user": {1063 "id": 80766,1064 "username": "coded_bias_007",1065 "name": "James",1066 "avatar_template": "/letter_avatar_proxy/v4/letter/c/dbc845/{size}.png",1067 "trust_level": 01068 }1069 }1070 ]1071 },1072 {1073 "fancy_title": "Faster RCNN maP not improving. Suggestions?",1074 "id": 212523,1075 "title": "Faster RCNN maP not improving. Suggestions?",1076 "slug": "faster-rcnn-map-not-improving-suggestions",1077 "posts_count": 1,1078 "reply_count": 0,1079 "highest_post_number": 1,1080 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/c/d/cd182af74b27ee1f734e000202f7c724fa56a41f_2_1024x422.png",1081 "created_at": "2024-11-04T22:21:03.523Z",1082 "last_posted_at": "2024-11-04T22:21:03.586Z",1083 "bumped": true,1084 "bumped_at": "2024-11-04T22:24:08.335Z",1085 "archetype": "regular",1086 "unseen": false,1087 "pinned": false,1088 "unpinned": null,1089 "visible": true,1090 "closed": false,1091 "archived": false,1092 "bookmarked": null,1093 "liked": null,1094 "tags_descriptions": {},1095 "like_count": 0,1096 "views": 111,1097 "category_id": 5,1098 "featured_link": null,1099 "has_accepted_answer": false,1100 "posters": [1101 {1102 "extras": "latest single",1103 "description": "Original Poster, Most Recent Poster",1104 "user": {1105 "id": 80684,1106 "username": "shanalikhan",1107 "name": "Shan Khan",1108 "avatar_template": "/user_avatar/discuss.pytorch.org/shanalikhan/{size}/73764_2.png",1109 "trust_level": 11110 }1111 }1112 ]1113 },1114 {1115 "fancy_title": "Testing not working while training working",1116 "id": 214707,1117 "title": "Testing not working while training working",1118 "slug": "testing-not-working-while-training-working",1119 "posts_count": 2,1120 "reply_count": 0,1121 "highest_post_number": 2,1122 "image_url": null,1123 "created_at": "2024-12-27T17:53:27.966Z",1124 "last_posted_at": "2024-12-27T18:26:32.135Z",1125 "bumped": true,1126 "bumped_at": "2024-12-27T18:26:32.135Z",1127 "archetype": "regular",1128 "unseen": false,1129 "pinned": false,1130 "unpinned": null,1131 "visible": true,1132 "closed": false,1133 "archived": false,1134 "bookmarked": null,1135 "liked": null,1136 "tags_descriptions": {},1137 "like_count": 0,1138 "views": 58,1139 "category_id": 5,1140 "featured_link": null,1141 "has_accepted_answer": false,1142 "posters": [1143 {1144 "extras": null,1145 "description": "Original Poster",1146 "user": {1147 "id": 75464,1148 "username": "Marina_Thalassini",1149 "name": "Marina Thalassini",1150 "avatar_template": "/user_avatar/discuss.pytorch.org/marina_thalassini/{size}/62322_2.png",1151 "trust_level": 11152 }1153 },1154 {1155 "extras": "latest",1156 "description": "Most Recent Poster",1157 "user": {1158 "id": 3534,1159 "username": "ptrblck",1160 "name": "",1161 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1162 "admin": true,1163 "moderator": true,1164 "trust_level": 21165 }1166 }1167 ]1168 },1169 {1170 "fancy_title": "Yolo model does not propose any targets during training",1171 "id": 215083,1172 "title": "Yolo model does not propose any targets during training",1173 "slug": "yolo-model-does-not-propose-any-targets-during-training",1174 "posts_count": 1,1175 "reply_count": 0,1176 "highest_post_number": 1,1177 "image_url": null,1178 "created_at": "2025-01-07T22:33:01.101Z",1179 "last_posted_at": "2025-01-07T22:33:01.148Z",1180 "bumped": true,1181 "bumped_at": "2025-01-07T22:33:01.148Z",1182 "archetype": "regular",1183 "unseen": false,1184 "pinned": false,1185 "unpinned": null,1186 "visible": true,1187 "closed": false,1188 "archived": false,1189 "bookmarked": null,1190 "liked": null,1191 "tags_descriptions": {},1192 "like_count": 0,1193 "views": 25,1194 "category_id": 5,1195 "featured_link": null,1196 "has_accepted_answer": false,1197 "posters": [1198 {1199 "extras": "latest single",1200 "description": "Original Poster, Most Recent Poster",