Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 206645,7 "name": "Stefano Savian",8 "username": "Stefano_Savian",9 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png",10 "created_at": "2020-06-25T09:13:24.931Z",11 "cooked": "<p>Dear all,</p>\n<p>I am experiencing a strange behavior of the dataloader when I import some custom modules at the beginning of my training script. After the custom scripts are loaded and when the training starts (or after few iterations) i encounter: \"RuntimeError: DataLoader worker (pid 7326) is killed by signal: Floating point exception. \" when the worker tries to get the new batch.</p>\n<p>If I don’t import everything runs fine, if I set num workers = 0, the network trains but extremely slowly.</p>\n<p>I have spent quite a lot of time trying to import my custom modules differently (at the beginning I was doing sys.append, but now I have changed everything to “import folder.subfolder.script_name”, which seems more correct. However this does not solve the problem.</p>\n<p>It is very hard to debug for me, could you give me some hints? <a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> or <a class=\"mention\" href=\"/u/alband\">@albanD</a> ?</p>\n<p>Kind Regards,<br>\nStefano</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 5,15 "updated_at": "2020-06-25T09:22:24.832Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 240,20 "reads": 9,21 "readers_count": 8,22 "score": 1201.8,23 "yours": false,24 "topic_id": 86855,25 "topic_slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",26 "display_username": "Stefano Savian",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 2,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 20931,48 "hidden": false,49 "trust_level": 2,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules/86855/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 206734,64 "name": "Alban D",65 "username": "albanD",66 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",67 "created_at": "2020-06-25T14:10:28.404Z",68 "cooked": "<p>Hi,</p>\n<p>This is surprising indeed.<br>\nWith 0 workers, does it goes through a full epoch without issues (even though it’s slow)? Could it be that one sample is problematic?</p>\n<p>I’m afraid I don’t have a silver bullet idea to debug this.<br>\nWhat I would do is like an ablation study. Remove stuff in your Dataset code until it stops crashing. (you can make the training loop a no-op and just load sample and discard them to make this run faster). I would be curious to know what is the minimal change that cause this!</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 5,72 "updated_at": "2020-06-25T14:10:28.404Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 3,77 "reads": 9,78 "readers_count": 8,79 "score": 21.8,80 "yours": false,81 "topic_id": 86855,82 "topic_slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",83 "display_username": "Alban D",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 211,105 "hidden": false,106 "trust_level": 4,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules/86855/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 206764,120 "name": "Stefano Savian",121 "username": "Stefano_Savian",122 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png",123 "created_at": "2020-06-25T14:54:02.406Z",124 "cooked": "<p>thanks so much for your fast response,</p>\n<p>I did not try to run a full epoch with 0 workers, because it is really slow. I will try to identify exactly where this happens in the code, and I will try to run a full epoch with 0 workers.</p>\n<p>I think the dataset is fine because this does not seem to happen when I remove my custom modules.<br>\nMy best guess is that since my custom scripts involve loading a module for inference, and the module is loaded in the <strong>init</strong> method of a class, this somehow interfere with the training script. Even though the loaded modules are only used after the training is completed.</p>\n<p>thanks again, I will come back to you!</p>\n<p>I will also try to</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 5,128 "updated_at": "2020-06-25T14:54:02.406Z",129 "reply_count": 1,130 "reply_to_post_number": 2,131 "quote_count": 0,132 "incoming_link_count": 2,133 "reads": 8,134 "readers_count": 7,135 "score": 61.6,136 "yours": false,137 "topic_id": 86855,138 "topic_slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",139 "display_username": "Stefano Savian",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 1,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": null,155 "reply_to_user": {156 "id": 211,157 "username": "albanD",158 "name": "Alban D",159 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png"160 },161 "bookmarked": false,162 "actions_summary": [163 {164 "id": 2,165 "count": 1166 }167 ],168 "moderator": false,169 "admin": false,170 "staff": false,171 "user_id": 20931,172 "hidden": false,173 "trust_level": 2,174 "deleted_at": null,175 "user_deleted": false,176 "edit_reason": null,177 "can_view_edit_history": true,178 "wiki": false,179 "post_url": "/t/runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules/86855/3",180 "can_accept_answer": false,181 "can_unaccept_answer": false,182 "accepted_answer": false,183 "topic_accepted_answer": null184 },185 {186 "id": 206849,187 "name": "Stefano Savian",188 "username": "Stefano_Savian",189 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png",190 "created_at": "2020-06-25T19:48:51.170Z",191 "cooked": "<p><a class=\"mention\" href=\"/u/alband\">@albanD</a>, I finally found the issue!</p>\n<p>I have used tensorboard for logging during training, I use this repository <a href=\"https://github.com/anderskm/exportTensorFlowLog/blob/master/exportTensorFlowLog.py\" rel=\"nofollow noopener\">TensorFlowLog</a> to export the logs. If I do a relative import of TensorFlowLog from the main.py everything is fine, however if I do a relative import of the same module from another file in another directory it creates problems. For another directory I mean a directory in the project but not a child directory of the main.py.<br>\nSimply, absolute import of TensorFlowLog solved the issue!</p>\n<p>EDIT: it actually did not solve the issue. I accidentally remove some lines during debugging. The issue is still present. My workaround is to import tensorboard.backend.everything at the very last part of the code, I really hate this solution but found nothing better.</p>\n<p>This bug, I think, was hard to detect: 1 failing silently during the importing phase, 2 generating “floating point error” during training, which seems completely unrelated to the actual issue.</p>\n<p>After little experience, I think that tensorboard is very buggy in my setup. I was thinking maybe It would be better to manually save logs by incrementally update a dataframe and save as it csv. I just need to to save some scalars.<br>\nWhat could be for you a simple and clean way to save training logs?(Loss, accuracy, validation…)</p>\n<p>Thanks so much,<br>\nStefano</p>",192 "post_number": 4,193 "post_type": 1,194 "posts_count": 5,195 "updated_at": "2020-07-03T13:07:39.981Z",196 "reply_count": 1,197 "reply_to_post_number": 3,198 "quote_count": 0,199 "incoming_link_count": 2,200 "reads": 6,201 "readers_count": 5,202 "score": 16.2,203 "yours": false,204 "topic_id": 86855,205 "topic_slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",206 "display_username": "Stefano Savian",207 "primary_group_name": null,208 "flair_name": null,209 "flair_url": null,210 "flair_bg_color": null,211 "flair_color": null,212 "flair_group_id": null,213 "badges_granted": [],214 "version": 2,215 "can_edit": false,216 "can_delete": false,217 "can_recover": false,218 "can_see_hidden_post": false,219 "can_wiki": false,220 "link_counts": [221 {222 "url": "https://github.com/anderskm/exportTensorFlowLog/blob/master/exportTensorFlowLog.py",223 "internal": false,224 "reflection": false,225 "title": "exportTensorFlowLog/exportTensorFlowLog.py at master · anderskm/exportTensorFlowLog · GitHub",226 "clicks": 0227 }228 ],229 "read": true,230 "user_title": null,231 "reply_to_user": {232 "id": 20931,233 "username": "Stefano_Savian",234 "name": "Stefano Savian",235 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png"236 },237 "bookmarked": false,238 "actions_summary": [],239 "moderator": false,240 "admin": false,241 "staff": false,242 "user_id": 20931,243 "hidden": false,244 "trust_level": 2,245 "deleted_at": null,246 "user_deleted": false,247 "edit_reason": null,248 "can_view_edit_history": true,249 "wiki": false,250 "post_url": "/t/runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules/86855/4",251 "can_accept_answer": false,252 "can_unaccept_answer": false,253 "accepted_answer": false,254 "topic_accepted_answer": null255 },256 {257 "id": 206852,258 "name": "Alban D",259 "username": "albanD",260 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",261 "created_at": "2020-06-25T19:53:44.896Z",262 "cooked": "<p>Nice catch. That indeed very far from anything I could have guessed haha</p>\n<p>I don’t have experience with the latest logging solutions <img src=\"https://discuss.pytorch.org/images/emoji/apple/confused.png?v=9\" title=\":confused:\" class=\"emoji\" alt=\":confused:\"><br>\nIf you only want to log numerical values though, you might want to check vanilla python options that might be more mature?</p>",263 "post_number": 5,264 "post_type": 1,265 "posts_count": 5,266 "updated_at": "2020-06-25T19:53:44.896Z",267 "reply_count": 0,268 "reply_to_post_number": 4,269 "quote_count": 0,270 "incoming_link_count": 2,271 "reads": 6,272 "readers_count": 5,273 "score": 26.2,274 "yours": false,275 "topic_id": 86855,276 "topic_slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",277 "display_username": "Alban D",278 "primary_group_name": null,279 "flair_name": null,280 "flair_url": null,281 "flair_bg_color": null,282 "flair_color": null,283 "flair_group_id": null,284 "badges_granted": [],285 "version": 1,286 "can_edit": false,287 "can_delete": false,288 "can_recover": false,289 "can_see_hidden_post": false,290 "can_wiki": false,291 "read": true,292 "user_title": "",293 "reply_to_user": {294 "id": 20931,295 "username": "Stefano_Savian",296 "name": "Stefano Savian",297 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png"298 },299 "bookmarked": false,300 "actions_summary": [301 {302 "id": 2,303 "count": 1304 }305 ],306 "moderator": true,307 "admin": true,308 "staff": true,309 "user_id": 211,310 "hidden": false,311 "trust_level": 4,312 "deleted_at": null,313 "user_deleted": false,314 "edit_reason": null,315 "can_view_edit_history": true,316 "wiki": false,317 "post_url": "/t/runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules/86855/5",318 "can_accept_answer": false,319 "can_unaccept_answer": false,320 "accepted_answer": false,321 "topic_accepted_answer": null322 }323 ],324 "stream": [325 206645,326 206734,327 206764,328 206849,329 206852330 ]331 },332 "timeline_lookup": [333 [334 1,335 1949336 ],337 [338 2,339 1948340 ]341 ],342 "suggested_topics": [343 {344 "fancy_title": "In GPU-memory compression",345 "id": 213987,346 "title": "In GPU-memory compression",347 "slug": "in-gpu-memory-compression",348 "posts_count": 1,349 "reply_count": 0,350 "highest_post_number": 1,351 "image_url": null,352 "created_at": "2024-12-09T11:15:32.161Z",353 "last_posted_at": "2024-12-09T11:15:32.219Z",354 "bumped": true,355 "bumped_at": "2024-12-09T11:15:32.219Z",356 "archetype": "regular",357 "unseen": false,358 "pinned": false,359 "unpinned": null,360 "visible": true,361 "closed": false,362 "archived": false,363 "bookmarked": null,364 "liked": null,365 "tags_descriptions": {},366 "like_count": 0,367 "views": 125,368 "category_id": 5,369 "featured_link": null,370 "has_accepted_answer": false,371 "posters": [372 {373 "extras": "latest single",374 "description": "Original Poster, Most Recent Poster",375 "user": {376 "id": 52896,377 "username": "Theophile_Champion",378 "name": "Theophile Champion",379 "avatar_template": "/user_avatar/discuss.pytorch.org/theophile_champion/{size}/30397_2.png",380 "trust_level": 1381 }382 }383 ]384 },385 {386 "fancy_title": "My predicted output ends up being a list of outputs",387 "id": 219642,388 "title": "My predicted output ends up being a list of outputs",389 "slug": "my-predicted-output-ends-up-being-a-list-of-outputs",390 "posts_count": 1,391 "reply_count": 0,392 "highest_post_number": 1,393 "image_url": null,394 "created_at": "2025-04-30T22:42:29.505Z",395 "last_posted_at": "2025-04-30T22:42:29.546Z",396 "bumped": true,397 "bumped_at": "2025-05-01T05:14:11.761Z",398 "archetype": "regular",399 "unseen": false,400 "pinned": false,401 "unpinned": null,402 "visible": true,403 "closed": false,404 "archived": false,405 "bookmarked": null,406 "liked": null,407 "tags_descriptions": {},408 "like_count": 0,409 "views": 32,410 "category_id": 5,411 "featured_link": null,412 "has_accepted_answer": false,413 "posters": [414 {415 "extras": "latest single",416 "description": "Original Poster, Most Recent Poster",417 "user": {418 "id": 84040,419 "username": "akulkarni3472",420 "name": "A Kulkarni",421 "avatar_template": "/user_avatar/discuss.pytorch.org/akulkarni3472/{size}/76809_2.png",422 "trust_level": 1423 }424 }425 ]426 },427 {428 "fancy_title": "I don’t no my forward pass and backward pass is too high in my code , that is about Forward/backward pass size (MB): 67652215240073.80",429 "id": 215676,430 "title": "I don't no my forward pass and backward pass is too high in my code , that is about Forward/backward pass size (MB): 67652215240073.80",431 "slug": "i-dont-no-my-forward-pass-and-backward-pass-is-too-high-in-my-code-that-is-about-forward-backward-pass-size-mb-67652215240073-80",432 "posts_count": 2,433 "reply_count": 0,434 "highest_post_number": 2,435 "image_url": null,436 "created_at": "2025-01-21T14:29:44.889Z",437 "last_posted_at": "2025-01-21T15:03:00.388Z",438 "bumped": true,439 "bumped_at": "2025-01-21T15:03:00.388Z",440 "archetype": "regular",441 "unseen": false,442 "pinned": false,443 "unpinned": null,444 "visible": true,445 "closed": false,446 "archived": false,447 "bookmarked": null,448 "liked": null,449 "tags_descriptions": {},450 "like_count": 0,451 "views": 39,452 "category_id": 5,453 "featured_link": null,454 "has_accepted_answer": false,455 "posters": [456 {457 "extras": null,458 "description": "Original Poster",459 "user": {460 "id": 82221,461 "username": "Goutham",462 "name": "Goutham",463 "avatar_template": "/user_avatar/discuss.pytorch.org/goutham/{size}/75223_2.png",464 "trust_level": 0465 }466 },467 {468 "extras": "latest",469 "description": "Most Recent Poster",470 "user": {471 "id": 3534,472 "username": "ptrblck",473 "name": "",474 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",475 "admin": true,476 "moderator": true,477 "trust_level": 2478 }479 }480 ]481 },482 {483 "fancy_title": "RuntimeError: Unsupported value kind: Tensor",484 "id": 218267,485 "title": "RuntimeError: Unsupported value kind: Tensor",486 "slug": "runtimeerror-unsupported-value-kind-tensor",487 "posts_count": 1,488 "reply_count": 0,489 "highest_post_number": 1,490 "image_url": null,491 "created_at": "2025-03-25T21:20:15.941Z",492 "last_posted_at": "2025-03-25T21:20:15.982Z",493 "bumped": true,494 "bumped_at": "2025-03-25T21:20:15.982Z",495 "archetype": "regular",496 "unseen": false,497 "pinned": false,498 "unpinned": null,499 "visible": true,500 "closed": false,501 "archived": false,502 "bookmarked": null,503 "liked": null,504 "tags_descriptions": {},505 "like_count": 0,506 "views": 64,507 "category_id": 5,508 "featured_link": null,509 "has_accepted_answer": false,510 "posters": [511 {512 "extras": "latest single",513 "description": "Original Poster, Most Recent Poster",514 "user": {515 "id": 82436,516 "username": "ettu",517 "name": "",518 "avatar_template": "/user_avatar/discuss.pytorch.org/ettu/{size}/75419_2.png",519 "trust_level": 1520 }521 }522 ]523 },524 {525 "fancy_title": "Fine tuning pretrained RestNet for grayscale image classification",526 "id": 221018,527 "title": "Fine tuning pretrained RestNet for grayscale image classification",528 "slug": "fine-tuning-pretrained-restnet-for-grayscale-image-classification",529 "posts_count": 2,530 "reply_count": 0,531 "highest_post_number": 2,532 "image_url": null,533 "created_at": "2025-06-24T11:03:10.058Z",534 "last_posted_at": "2025-06-25T14:22:46.038Z",535 "bumped": true,536 "bumped_at": "2025-06-25T14:22:46.038Z",537 "archetype": "regular",538 "unseen": false,539 "pinned": false,540 "unpinned": null,541 "visible": true,542 "closed": false,543 "archived": false,544 "bookmarked": null,545 "liked": null,546 "tags_descriptions": {},547 "like_count": 0,548 "views": 70,549 "category_id": 5,550 "featured_link": null,551 "has_accepted_answer": false,552 "posters": [553 {554 "extras": null,555 "description": "Original Poster",556 "user": {557 "id": 84808,558 "username": "abir",559 "name": "",560 "avatar_template": "/user_avatar/discuss.pytorch.org/abir/{size}/77461_2.png",561 "trust_level": 1562 }563 },564 {565 "extras": "latest",566 "description": "Most Recent Poster",567 "user": {568 "id": 3534,569 "username": "ptrblck",570 "name": "",571 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",572 "admin": true,573 "moderator": true,574 "trust_level": 2575 }576 }577 ]578 }579 ],580 "tags_descriptions": {},581 "fancy_title": "RuntimeError: DataLoader worker killed when importing some custom modules",582 "id": 86855,583 "title": "RuntimeError: DataLoader worker killed when importing some custom modules",584 "posts_count": 5,585 "created_at": "2020-06-25T09:13:24.879Z",586 "views": 880,587 "reply_count": 3,588 "like_count": 2,589 "last_posted_at": "2020-06-25T19:53:44.896Z",590 "visible": true,591 "closed": false,592 "archived": false,593 "has_summary": false,594 "archetype": "regular",595 "slug": "runtimeerror-dataloader-worker-killed-when-importing-some-custom-modules",596 "category_id": 5,597 "word_count": 650,598 "deleted_at": null,599 "user_id": 20931,600 "featured_link": null,601 "pinned_globally": false,602 "pinned_at": null,603 "pinned_until": null,604 "image_url": null,605 "slow_mode_seconds": 0,606 "draft": null,607 "draft_key": "topic_86855",608 "draft_sequence": null,609 "unpinned": null,610 "pinned": false,611 "current_post_number": 1,612 "highest_post_number": 5,613 "deleted_by": null,614 "actions_summary": [615 {616 "id": 4,617 "count": 0,618 "hidden": false,619 "can_act": false620 },621 {622 "id": 8,623 "count": 0,624 "hidden": false,625 "can_act": false626 },627 {628 "id": 10,629 "count": 0,630 "hidden": false,631 "can_act": false632 },633 {634 "id": 7,635 "count": 0,636 "hidden": false,637 "can_act": false638 }639 ],640 "chunk_size": 20,641 "bookmarked": false,642 "topic_timer": null,643 "message_bus_last_id": 0,644 "participant_count": 2,645 "show_read_indicator": false,646 "thumbnails": null,647 "slow_mode_enabled_until": null,648 "can_vote": false,649 "vote_count": 0,650 "user_voted": false,651 "discourse_zendesk_plugin_zendesk_id": null,652 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",653 "details": {654 "can_edit": false,655 "notification_level": 1,656 "participants": [657 {658 "id": 20931,659 "username": "Stefano_Savian",660 "name": "Stefano Savian",661 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png",662 "post_count": 3,663 "primary_group_name": null,664 "flair_name": null,665 "flair_url": null,666 "flair_color": null,667 "flair_bg_color": null,668 "flair_group_id": null,669 "trust_level": 2670 },671 {672 "id": 211,673 "username": "albanD",674 "name": "Alban D",675 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",676 "post_count": 2,677 "primary_group_name": null,678 "flair_name": null,679 "flair_url": null,680 "flair_color": null,681 "flair_bg_color": null,682 "flair_group_id": null,683 "admin": true,684 "moderator": true,685 "trust_level": 4686 }687 ],688 "created_by": {689 "id": 20931,690 "username": "Stefano_Savian",691 "name": "Stefano Savian",692 "avatar_template": "/user_avatar/discuss.pytorch.org/stefano_savian/{size}/14377_2.png"693 },694 "last_poster": {695 "id": 211,696 "username": "albanD",697 "name": "Alban D",698 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png"699 }700 },701 "bookmarks": []702 },703 {704 "post_stream": {705 "posts": [706 {707 "id": 178328,708 "name": "",709 "username": "shrutee",710 "avatar_template": "/user_avatar/discuss.pytorch.org/shrutee/{size}/22276_2.png",711 "created_at": "2020-04-01T12:08:07.950Z",712 "cooked": "<p>I am using <code>torch.cuda.amp</code> for mixed precision.</p>\n<p>My forward pass calls many functions with their own <code>forward</code> passes.<br>\nI tried to decorate all the forward passes in the subsequent functions with <code>torch.cuda.amp.autocast(enabled=True)</code> but the error persists.</p>\n<p>Forward pass:</p>\n<pre><code class=\"lang-auto\">with torch.cuda.amp.autocast(enabled=True):\n h, chunk, preds, labels = model.forward(batch, alphaSG, device)\n label = labels\n for worker in model.classification_workers:\n loss = worker.loss_weight * worker.loss(preds[worker.name], label[worker.name])\n losses[worker.name] = loss\n tot_loss += loss\n\n for worker in model.regression_workers:\n loss = worker.loss_weight * worker.loss(preds[worker.name], label[worker.name])\n losses[worker.name] = loss\n tot_loss += loss\n</code></pre>\n<p>Error I am getting:</p>\n<pre><code class=\"lang-auto\">RuntimeError Traceback (most recent call last)\n<ipython-input-10-f4ce5bf32b0d> in <module>()\n 2327 \n 2328 with torch.cuda.amp.autocast(enabled=True):\n-> 2329 h, chunk, preds, labels = model.forward(batch, alphaSG, device)\n 2330 label = labels\n 2331 for worker in model.classification_workers:\n\n10 frames\n/usr/local/lib/python3.6/dist-packages/torch/cuda/amp/autocast_mode.py in decorate_autocast(*args, **kwargs)\n 133 def decorate_autocast(*args, **kwargs):\n 134 with self:\n--> 135 return func(*args, **kwargs)\n 136 return decorate_autocast\n 137 \n\n<ipython-input-10-f4ce5bf32b0d> in forward(self, x, alpha, device)\n 1945 # remove key if it exists\n 1946 x_.pop('cchunk', None)\n-> 1947 h = self.frontend(x_, device)\n 1948 if len(h) > 1:\n 1949 assert len(h) == 2, len(h)\n\n/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)\n 556 result = self._slow_forward(*input, **kwargs)\n 557 else:\n--> 558 result = self.forward(*input, **kwargs)\n 559 for hook in self._forward_hooks.values():\n 560 hook_result = hook(self, input, result)\n\n/usr/local/lib/python3.6/dist-packages/torch/cuda/amp/autocast_mode.py in decorate_autocast(*args, **kwargs)\n 133 def decorate_autocast(*args, **kwargs):\n 134 with self:\n--> 135 return func(*args, **kwargs)\n 136 return decorate_autocast\n 137 \n\n<ipython-input-10-f4ce5bf32b0d> in forward(self, batch, device, mode)\n 1827 dskips = []\n 1828 for n, block in enumerate(self.blocks):\n-> 1829 h = block(h)\n 1830 if denseskips and (n + 1) < len(self.blocks):\n 1831 # denseskips happen til the last but one layer\n\n/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)\n 556 result = self._slow_forward(*input, **kwargs)\n 557 else:\n--> 558 result = self.forward(*input, **kwargs)\n 559 for hook in self._forward_hooks.values():\n 560 hook_result = hook(self, input, result)\n\n/usr/local/lib/python3.6/dist-packages/torch/cuda/amp/autocast_mode.py in decorate_autocast(*args, **kwargs)\n 133 def decorate_autocast(*args, **kwargs):\n 134 with self:\n--> 135 return func(*args, **kwargs)\n 136 return decorate_autocast\n 137 \n\n<ipython-input-10-f4ce5bf32b0d> in forward(self, x)\n 1494 P = (pad, pad)\n 1495 x = F.pad(x, P, mode=self.pad_mode)\n-> 1496 h = self.conv(x)\n 1497 if hasattr(self, 'norm'):\n 1498 h = forward_norm(h, self.norm)\n\n/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)\n 556 result = self._slow_forward(*input, **kwargs)\n 557 else:\n--> 558 result = self.forward(*input, **kwargs)\n 559 for hook in self._forward_hooks.values():\n 560 hook_result = hook(self, input, result)\n\n/usr/local/lib/python3.6/dist-packages/torch/cuda/amp/autocast_mode.py in decorate_autocast(*args, **kwargs)\n 133 def decorate_autocast(*args, **kwargs):\n 134 with self:\n--> 135 return func(*args, **kwargs)\n 136 return decorate_autocast\n 137 \n\n<ipython-input-10-f4ce5bf32b0d> in forward(self, waveforms)\n 1334 band=(high-low)[:,0]\n 1335 \n-> 1336 f_times_t_low = torch.matmul(low, self.n_)\n 1337 f_times_t_high = torch.matmul(high, self.n_)\n</code></pre>",713 "post_number": 1,714 "post_type": 1,715 "posts_count": 7,716 "updated_at": "2020-04-01T12:08:07.950Z",717 "reply_count": 0,718 "reply_to_post_number": null,719 "quote_count": 0,720 "incoming_link_count": 5963,721 "reads": 136,722 "readers_count": 135,723 "score": 29832.2,724 "yours": false,725 "topic_id": 74996,726 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",727 "display_username": "",728 "primary_group_name": null,729 "flair_name": null,730 "flair_url": null,731 "flair_bg_color": null,732 "flair_color": null,733 "flair_group_id": null,734 "badges_granted": [],735 "version": 1,736 "can_edit": false,737 "can_delete": false,738 "can_recover": false,739 "can_see_hidden_post": false,740 "can_wiki": false,741 "read": true,742 "user_title": null,743 "bookmarked": false,744 "actions_summary": [745 {746 "id": 2,747 "count": 1748 }749 ],750 "moderator": false,751 "admin": false,752 "staff": false,753 "user_id": 25261,754 "hidden": false,755 "trust_level": 1,756 "deleted_at": null,757 "user_deleted": false,758 "edit_reason": null,759 "can_view_edit_history": true,760 "wiki": false,761 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/1",762 "can_accept_answer": false,763 "can_unaccept_answer": false,764 "accepted_answer": false,765 "topic_accepted_answer": true,766 "can_vote": false767 },768 {769 "id": 178487,770 "name": "",771 "username": "ptrblck",772 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",773 "created_at": "2020-04-02T01:58:58.100Z",774 "cooked": "<p>Could you post a code snippet to reproduce this issue, please?</p>",775 "post_number": 2,776 "post_type": 1,777 "posts_count": 7,778 "updated_at": "2020-04-02T01:58:58.100Z",779 "reply_count": 0,780 "reply_to_post_number": null,781 "quote_count": 0,782 "incoming_link_count": 27,783 "reads": 123,784 "readers_count": 122,785 "score": 159.6,786 "yours": false,787 "topic_id": 74996,788 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",789 "display_username": "",790 "primary_group_name": null,791 "flair_name": null,792 "flair_url": null,793 "flair_bg_color": null,794 "flair_color": null,795 "flair_group_id": null,796 "badges_granted": [],797 "version": 1,798 "can_edit": false,799 "can_delete": false,800 "can_recover": false,801 "can_see_hidden_post": false,802 "can_wiki": false,803 "read": true,804 "user_title": "",805 "bookmarked": false,806 "actions_summary": [],807 "moderator": true,808 "admin": true,809 "staff": true,810 "user_id": 3534,811 "hidden": false,812 "trust_level": 2,813 "deleted_at": null,814 "user_deleted": false,815 "edit_reason": null,816 "can_view_edit_history": true,817 "wiki": false,818 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/2",819 "can_accept_answer": false,820 "can_unaccept_answer": false,821 "accepted_answer": false,822 "topic_accepted_answer": true823 },824 {825 "id": 178544,826 "name": "",827 "username": "shrutee",828 "avatar_template": "/user_avatar/discuss.pytorch.org/shrutee/{size}/22276_2.png",829 "created_at": "2020-04-02T04:48:44.223Z",830 "cooked": "<p>Hi,<br>\nHere is the <a href=\"https://colab.research.google.com/gist/MittalShruti/c4d9f7292b10f2eb6c6130f69529bfd6/mixed-precision-torch-cuda-amp.ipynb?authuser=3\" rel=\"nofollow noopener\">code</a></p>\n<p>this is 115 lines of code. Please let me know if this wouldn’t help.</p>",831 "post_number": 3,832 "post_type": 1,833 "posts_count": 7,834 "updated_at": "2020-04-02T04:48:44.223Z",835 "reply_count": 1,836 "reply_to_post_number": null,837 "quote_count": 0,838 "incoming_link_count": 24,839 "reads": 123,840 "readers_count": 122,841 "score": 149.6,842 "yours": false,843 "topic_id": 74996,844 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",845 "display_username": "",846 "primary_group_name": null,847 "flair_name": null,848 "flair_url": null,849 "flair_bg_color": null,850 "flair_color": null,851 "flair_group_id": null,852 "badges_granted": [],853 "version": 1,854 "can_edit": false,855 "can_delete": false,856 "can_recover": false,857 "can_see_hidden_post": false,858 "can_wiki": false,859 "link_counts": [860 {861 "url": "https://colab.research.google.com/gist/MittalShruti/c4d9f7292b10f2eb6c6130f69529bfd6/mixed-precision-torch-cuda-amp.ipynb?authuser=3",862 "internal": false,863 "reflection": false,864 "clicks": 82865 }866 ],867 "read": true,868 "user_title": null,869 "bookmarked": false,870 "actions_summary": [],871 "moderator": false,872 "admin": false,873 "staff": false,874 "user_id": 25261,875 "hidden": false,876 "trust_level": 1,877 "deleted_at": null,878 "user_deleted": false,879 "edit_reason": null,880 "can_view_edit_history": true,881 "wiki": false,882 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/3",883 "can_accept_answer": false,884 "can_unaccept_answer": false,885 "accepted_answer": false,886 "topic_accepted_answer": true887 },888 {889 "id": 178564,890 "name": "",891 "username": "ptrblck",892 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",893 "created_at": "2020-04-02T07:06:53.885Z",894 "cooked": "<p>The error points towards a device mismatch:</p>\n<pre><code class=\"lang-python\">RuntimeError: Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mm\n</code></pre>\n<p><code>amp</code> needs a GPU to run properly, so you would need to call <code>.to('cuda')</code> on the model and input.<br>\nAlso, you might need to register <code>self.n_</code> as a buffer via:</p>\n<pre><code class=\"lang-python\">self.register_buffer('n_', ...)\n</code></pre>\n<p>so that it will be also pushed to the device.</p>\n<p>Your code doesn’t provide the definition of <code>format_frontend_output</code> so that I cannot verify, if it’s working after these fixes.</p>",895 "post_number": 4,896 "post_type": 1,897 "posts_count": 7,898 "updated_at": "2020-04-02T08:01:38.019Z",899 "reply_count": 0,900 "reply_to_post_number": 3,901 "quote_count": 0,902 "incoming_link_count": 264,903 "reads": 121,904 "readers_count": 120,905 "score": 1344.2,906 "yours": false,907 "topic_id": 74996,908 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",909 "display_username": "",910 "primary_group_name": null,911 "flair_name": null,912 "flair_url": null,913 "flair_bg_color": null,914 "flair_color": null,915 "flair_group_id": null,916 "badges_granted": [],917 "version": 1,918 "can_edit": false,919 "can_delete": false,920 "can_recover": false,921 "can_see_hidden_post": false,922 "can_wiki": false,923 "read": true,924 "user_title": "",925 "reply_to_user": {926 "id": 25261,927 "username": "shrutee",928 "name": "",929 "avatar_template": "/user_avatar/discuss.pytorch.org/shrutee/{size}/22276_2.png"930 },931 "bookmarked": false,932 "actions_summary": [],933 "moderator": true,934 "admin": true,935 "staff": true,936 "user_id": 3534,937 "hidden": false,938 "trust_level": 2,939 "deleted_at": null,940 "user_deleted": false,941 "edit_reason": null,942 "can_view_edit_history": true,943 "wiki": false,944 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/4",945 "can_accept_answer": false,946 "can_unaccept_answer": false,947 "accepted_answer": true,948 "topic_accepted_answer": true949 },950 {951 "id": 178574,952 "name": "",953 "username": "shrutee",954 "avatar_template": "/user_avatar/discuss.pytorch.org/shrutee/{size}/22276_2.png",955 "created_at": "2020-04-02T08:01:34.518Z",956 "cooked": "<p>Thank You!! this solves the error.</p>\n<p>Two points -</p>\n<ol>\n<li>\n<p>Where/how did you get the device mismatch error?</p>\n</li>\n<li>\n<p><code>self.n_</code> is already on device, so why use register_buffer again? There are many other variables in the actual code, how do you decide if a parameter should be registered as a buffer or not? Here, in this code, e.g. I have <code>self.window_</code> along with <code>self.n_</code>; why not register it as buffer as well?</p>\n</li>\n</ol>",957 "post_number": 5,958 "post_type": 1,959 "posts_count": 7,960 "updated_at": "2020-04-02T08:01:34.518Z",961 "reply_count": 1,962 "reply_to_post_number": null,963 "quote_count": 0,964 "incoming_link_count": 28,965 "reads": 116,966 "readers_count": 115,967 "score": 168.2,968 "yours": false,969 "topic_id": 74996,970 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",971 "display_username": "",972 "primary_group_name": null,973 "flair_name": null,974 "flair_url": null,975 "flair_bg_color": null,976 "flair_color": null,977 "flair_group_id": null,978 "badges_granted": [],979 "version": 1,980 "can_edit": false,981 "can_delete": false,982 "can_recover": false,983 "can_see_hidden_post": false,984 "can_wiki": false,985 "read": true,986 "user_title": null,987 "bookmarked": false,988 "actions_summary": [],989 "moderator": false,990 "admin": false,991 "staff": false,992 "user_id": 25261,993 "hidden": false,994 "trust_level": 1,995 "deleted_at": null,996 "user_deleted": false,997 "edit_reason": null,998 "can_view_edit_history": true,999 "wiki": false,1000 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/5",1001 "can_accept_answer": false,1002 "can_unaccept_answer": false,1003 "accepted_answer": false,1004 "topic_accepted_answer": true1005 },1006 {1007 "id": 178576,1008 "name": "",1009 "username": "ptrblck",1010 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1011 "created_at": "2020-04-02T08:26:52.915Z",1012 "cooked": "<p>I’m glad it solved the error! <img src=\"https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9\" title=\":wink:\" class=\"emoji\" alt=\":wink:\"></p>\n<ol>\n<li>\n<p>The error was raised in my environment. If you have executed the script in Colab, the error message might have been lost?</p>\n</li>\n<li>\n<p>Usually, every tensor, which should be pushed to the same device as the model parameters, but doesn’t require gradients, should be registered as a buffer. On the other hand, if the tensor requires gradients, use <code>nn.Parameter</code>.</p>\n</li>\n</ol>",1013 "post_number": 6,1014 "post_type": 1,1015 "posts_count": 7,1016 "updated_at": "2020-04-02T08:26:52.915Z",1017 "reply_count": 1,1018 "reply_to_post_number": 5,1019 "quote_count": 0,1020 "incoming_link_count": 19,1021 "reads": 109,1022 "readers_count": 108,1023 "score": 136.8,1024 "yours": false,1025 "topic_id": 74996,1026 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",1027 "display_username": "",1028 "primary_group_name": null,1029 "flair_name": null,1030 "flair_url": null,1031 "flair_bg_color": null,1032 "flair_color": null,1033 "flair_group_id": null,1034 "badges_granted": [],1035 "version": 1,1036 "can_edit": false,1037 "can_delete": false,1038 "can_recover": false,1039 "can_see_hidden_post": false,1040 "can_wiki": false,1041 "read": true,1042 "user_title": "",1043 "reply_to_user": {1044 "id": 25261,1045 "username": "shrutee",1046 "name": "",1047 "avatar_template": "/user_avatar/discuss.pytorch.org/shrutee/{size}/22276_2.png"1048 },1049 "bookmarked": false,1050 "actions_summary": [1051 {1052 "id": 2,1053 "count": 11054 }1055 ],1056 "moderator": true,1057 "admin": true,1058 "staff": true,1059 "user_id": 3534,1060 "hidden": false,1061 "trust_level": 2,1062 "deleted_at": null,1063 "user_deleted": false,1064 "edit_reason": null,1065 "can_view_edit_history": true,1066 "wiki": false,1067 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/6",1068 "can_accept_answer": false,1069 "can_unaccept_answer": false,1070 "accepted_answer": false,1071 "topic_accepted_answer": true1072 },1073 {1074 "id": 206851,1075 "name": "Kushagra Bhatia",1076 "username": "Kushagra_Bhatia",1077 "avatar_template": "/user_avatar/discuss.pytorch.org/kushagra_bhatia/{size}/25826_2.png",1078 "created_at": "2020-06-25T19:51:43.620Z",1079 "cooked": "<p>Can you please help me out with this I am facing a similar problem. Getting the error:<br>\n<code>Expected object of scalar type Float but got scalar type Half for argument #2 'mat2' in call to _th_mm</code></p>\n<p>Link to my problem: <a href=\"https://discuss.pytorch.org/t/expected-object-of-device-type-cuda-but-got-device-type-cpu-for-argument-1-self-in-call-to-th-mm/86921/13\" class=\"inline-onebox\">Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mm</a></p>\n<p>Thanks!</p>",1080 "post_number": 7,1081 "post_type": 1,1082 "posts_count": 7,1083 "updated_at": "2020-06-25T19:51:43.620Z",1084 "reply_count": 0,1085 "reply_to_post_number": 6,1086 "quote_count": 0,1087 "incoming_link_count": 20,1088 "reads": 93,1089 "readers_count": 92,1090 "score": 118.6,1091 "yours": false,1092 "topic_id": 74996,1093 "topic_slug": "runtimeerror-expected-scalar-type-float-but-found-half",1094 "display_username": "Kushagra Bhatia",1095 "primary_group_name": null,1096 "flair_name": null,1097 "flair_url": null,1098 "flair_bg_color": null,1099 "flair_color": null,1100 "flair_group_id": null,1101 "badges_granted": [],1102 "version": 1,1103 "can_edit": false,1104 "can_delete": false,1105 "can_recover": false,1106 "can_see_hidden_post": false,1107 "can_wiki": false,1108 "link_counts": [1109 {1110 "url": "https://discuss.pytorch.org/t/expected-object-of-device-type-cuda-but-got-device-type-cpu-for-argument-1-self-in-call-to-th-mm/86921/13",1111 "internal": true,1112 "reflection": false,1113 "title": "Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mm",1114 "clicks": 3001115 }1116 ],1117 "read": true,1118 "user_title": null,1119 "reply_to_user": {1120 "id": 3534,1121 "username": "ptrblck",1122 "name": "",1123 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"1124 },1125 "bookmarked": false,1126 "actions_summary": [],1127 "moderator": false,1128 "admin": false,1129 "staff": false,1130 "user_id": 33316,1131 "hidden": false,1132 "trust_level": 2,1133 "deleted_at": null,1134 "user_deleted": false,1135 "edit_reason": null,1136 "can_view_edit_history": true,1137 "wiki": false,1138 "post_url": "/t/runtimeerror-expected-scalar-type-float-but-found-half/74996/7",1139 "can_accept_answer": false,1140 "can_unaccept_answer": false,1141 "accepted_answer": false,1142 "topic_accepted_answer": true1143 }1144 ],1145 "stream": [1146 178328,1147 178487,1148 178544,1149 178564,1150 178574,1151 178576,1152 2068511153 ]1154 },1155 "timeline_lookup": [1156 [1157 1,1158 20331159 ],1160 [1161 7,1162 19481163 ]1164 ],1165 "suggested_topics": [1166 {1167 "fancy_title": "(torch.__version__) 2.7.0+cu126",1168 "id": 219650,1169 "title": "(torch.__version__) 2.7.0+cu126",1170 "slug": "torch-version-2-7-0-cu126",1171 "posts_count": 2,1172 "reply_count": 0,1173 "highest_post_number": 2,1174 "image_url": null,1175 "created_at": "2025-05-01T03:29:05.104Z",1176 "last_posted_at": "2025-05-01T11:58:25.492Z",1177 "bumped": true,1178 "bumped_at": "2025-05-01T11:58:25.492Z",1179 "archetype": "regular",1180 "unseen": false,1181 "pinned": false,1182 "unpinned": null,1183 "visible": true,1184 "closed": false,1185 "archived": false,1186 "bookmarked": null,1187 "liked": null,1188 "tags_descriptions": {},1189 "like_count": 0,1190 "views": 262,1191 "category_id": 1,1192 "featured_link": null,1193 "has_accepted_answer": false,1194 "posters": [1195 {1196 "extras": null,1197 "description": "Original Poster",1198 "user": {1199 "id": 84099,1200 "username": "Ghada_Almousa",