Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 461388,7 "name": "Raul Ciotescu",8 "username": "raulc",9 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png",10 "created_at": "2024-12-13T14:23:27.754Z",11 "cooked": "<p>the model uses “with sdp_kernel(enable_math=True, enable_flash=True, enable_mem_efficient=True):”</p>\n<p>when calling compile on this model, it generated the following error:</p>\n<blockquote>\n<p>torch._dynamo.exc.Unsupported: torch.* op returned non-Tensor _GeneratorContextManager call_function <function sdp_kernel at 0x7e3c02b79fc0></p>\n</blockquote>\n<p>how can i use sdp_kernel with compile</p>\n<p>thanks</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 5,15 "updated_at": "2024-12-13T14:23:27.754Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 272,20 "reads": 8,21 "readers_count": 7,22 "score": 1326.6,23 "yours": false,24 "topic_id": 214200,25 "topic_slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",26 "display_username": "Raul Ciotescu",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 75496,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work/214200/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 461396,64 "name": "",65 "username": "ptrblck",66 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",67 "created_at": "2024-12-13T18:57:23.580Z",68 "cooked": "<p><code>torch.compile</code> works for me but raises the deprecation warning:</p>\n<pre data-code-wrap=\"python\"><code class=\"lang-python\">query = torch.rand(64, 12, 77, 64, dtype=torch.float16, device=\"cuda\")\nkey = torch.rand(64, 12, 77, 64, dtype=torch.float16, device=\"cuda\")\nvalue = torch.rand(64, 12, 77, 64, dtype=torch.float16, device=\"cuda\")\nattn_mask = torch.rand(64, 1, 77, 77, dtype=torch.float16, device=\"cuda\")\n\n\ndef fun(query, key, value, attn_mask):\n with torch.backends.cuda.sdp_kernel(enable_math=True, enable_flash=True, enable_mem_efficient=True):\n out = torch.nn.functional.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask)\n return out\n\nout = fun(query, key, value, attn_mask)\n# FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature.\n\nfun_compiled = torch.compile(fun)\nout = fun_compiled(query, key, value, attn_mask)\n# FutureWarning: `torch.backends.cuda.sdp_kernel()` is deprecated. In the future, this context manager will be removed. Please see `torch.nn.attention.sdpa_kernel()` for the new context manager, with updated signature.\n</code></pre>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 5,72 "updated_at": "2024-12-13T18:57:23.580Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 2,77 "reads": 9,78 "readers_count": 8,79 "score": 31.8,80 "yours": false,81 "topic_id": 214200,82 "topic_slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",83 "display_username": "",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [101 {102 "id": 2,103 "count": 1104 }105 ],106 "moderator": true,107 "admin": true,108 "staff": true,109 "user_id": 3534,110 "hidden": false,111 "trust_level": 2,112 "deleted_at": null,113 "user_deleted": false,114 "edit_reason": null,115 "can_view_edit_history": true,116 "wiki": false,117 "post_url": "/t/compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work/214200/2",118 "can_accept_answer": false,119 "can_unaccept_answer": false,120 "accepted_answer": false,121 "topic_accepted_answer": null122 },123 {124 "id": 461411,125 "name": "Brian Hirsh",126 "username": "bdhirsh",127 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png",128 "created_at": "2024-12-13T23:55:19.892Z",129 "cooked": "<p>Can you move the context manager so it is outside of the compiled region? E.g</p>\n<p>With sdp_kerbel(…):<br>\nout = compiled_model(inp)</p>\n<p>Although cc <a class=\"mention\" href=\"/u/drisspg\">@drisspg</a> (we should make sure the new, non-deprecated context manager works inside of a compiled region)</p>",130 "post_number": 3,131 "post_type": 1,132 "posts_count": 5,133 "updated_at": "2024-12-13T23:55:19.892Z",134 "reply_count": 1,135 "reply_to_post_number": null,136 "quote_count": 0,137 "incoming_link_count": 3,138 "reads": 7,139 "readers_count": 6,140 "score": 36.4,141 "yours": false,142 "topic_id": 214200,143 "topic_slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",144 "display_username": "Brian Hirsh",145 "primary_group_name": null,146 "flair_name": null,147 "flair_url": null,148 "flair_bg_color": null,149 "flair_color": null,150 "flair_group_id": null,151 "badges_granted": [],152 "version": 1,153 "can_edit": false,154 "can_delete": false,155 "can_recover": false,156 "can_see_hidden_post": false,157 "can_wiki": false,158 "read": true,159 "user_title": null,160 "bookmarked": false,161 "actions_summary": [162 {163 "id": 2,164 "count": 1165 }166 ],167 "moderator": false,168 "admin": false,169 "staff": false,170 "user_id": 41997,171 "hidden": false,172 "trust_level": 2,173 "deleted_at": null,174 "user_deleted": false,175 "edit_reason": null,176 "can_view_edit_history": true,177 "wiki": false,178 "post_url": "/t/compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work/214200/3",179 "can_accept_answer": false,180 "can_unaccept_answer": false,181 "accepted_answer": false,182 "topic_accepted_answer": null183 },184 {185 "id": 461695,186 "name": "Raul Ciotescu",187 "username": "raulc",188 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png",189 "created_at": "2024-12-18T20:23:07.399Z",190 "cooked": "<p>sorry, forgot to mention that it was with fullgraph.</p>\n<blockquote>\n<p>fun_compiled = torch.compile(fun, fullgraph=True)</p>\n</blockquote>\n<p>it yields:</p>\n<blockquote>\n<p>File “/home/…/env/lib/python3.12/site-packages/torch/_dynamo/exc.py”, line 297, in unimplemented<br>\nraise Unsupported(msg, case_name=case_name)<br>\ntorch._dynamo.exc.Unsupported: torch.* op returned non-Tensor _GeneratorContextManager call_function <function sdp_kernel at 0x7fa07c0c8400></p>\n<p>from user code:<br>\nFile “/home/…/test.py”, line 10, in fun<br>\nwith torch.backends.cuda.sdp_kernel(enable_math=True, enable_flash=True, enable_mem_efficient=True):</p>\n</blockquote>",191 "post_number": 4,192 "post_type": 1,193 "posts_count": 5,194 "updated_at": "2024-12-18T20:23:07.399Z",195 "reply_count": 0,196 "reply_to_post_number": 2,197 "quote_count": 0,198 "incoming_link_count": 1,199 "reads": 6,200 "readers_count": 5,201 "score": 6.2,202 "yours": false,203 "topic_id": 214200,204 "topic_slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",205 "display_username": "Raul Ciotescu",206 "primary_group_name": null,207 "flair_name": null,208 "flair_url": null,209 "flair_bg_color": null,210 "flair_color": null,211 "flair_group_id": null,212 "badges_granted": [],213 "version": 1,214 "can_edit": false,215 "can_delete": false,216 "can_recover": false,217 "can_see_hidden_post": false,218 "can_wiki": false,219 "read": true,220 "user_title": null,221 "reply_to_user": {222 "id": 3534,223 "username": "ptrblck",224 "name": "",225 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"226 },227 "bookmarked": false,228 "actions_summary": [],229 "moderator": false,230 "admin": false,231 "staff": false,232 "user_id": 75496,233 "hidden": false,234 "trust_level": 1,235 "deleted_at": null,236 "user_deleted": false,237 "edit_reason": null,238 "can_view_edit_history": true,239 "wiki": false,240 "post_url": "/t/compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work/214200/4",241 "can_accept_answer": false,242 "can_unaccept_answer": false,243 "accepted_answer": false,244 "topic_accepted_answer": null245 },246 {247 "id": 461696,248 "name": "Raul Ciotescu",249 "username": "raulc",250 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png",251 "created_at": "2024-12-18T20:23:48.812Z",252 "cooked": "<p>i guess i can do that as a workaround. will try.</p>",253 "post_number": 5,254 "post_type": 1,255 "posts_count": 5,256 "updated_at": "2024-12-18T20:23:48.812Z",257 "reply_count": 0,258 "reply_to_post_number": 3,259 "quote_count": 0,260 "incoming_link_count": 0,261 "reads": 5,262 "readers_count": 4,263 "score": 1.0,264 "yours": false,265 "topic_id": 214200,266 "topic_slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",267 "display_username": "Raul Ciotescu",268 "primary_group_name": null,269 "flair_name": null,270 "flair_url": null,271 "flair_bg_color": null,272 "flair_color": null,273 "flair_group_id": null,274 "badges_granted": [],275 "version": 1,276 "can_edit": false,277 "can_delete": false,278 "can_recover": false,279 "can_see_hidden_post": false,280 "can_wiki": false,281 "read": true,282 "user_title": null,283 "reply_to_user": {284 "id": 41997,285 "username": "bdhirsh",286 "name": "Brian Hirsh",287 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png"288 },289 "bookmarked": false,290 "actions_summary": [],291 "moderator": false,292 "admin": false,293 "staff": false,294 "user_id": 75496,295 "hidden": false,296 "trust_level": 1,297 "deleted_at": null,298 "user_deleted": false,299 "edit_reason": null,300 "can_view_edit_history": true,301 "wiki": false,302 "post_url": "/t/compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work/214200/5",303 "can_accept_answer": false,304 "can_unaccept_answer": false,305 "accepted_answer": false,306 "topic_accepted_answer": null307 }308 ],309 "stream": [310 461388,311 461396,312 461411,313 461695,314 461696315 ]316 },317 "timeline_lookup": [318 [319 1,320 316321 ],322 [323 4,324 311325 ]326 ],327 "suggested_topics": [328 {329 "fancy_title": "Dynamic slicing torch.export",330 "id": 212889,331 "title": "Dynamic slicing torch.export",332 "slug": "dynamic-slicing-torch-export",333 "posts_count": 3,334 "reply_count": 0,335 "highest_post_number": 3,336 "image_url": null,337 "created_at": "2024-11-12T19:40:42.004Z",338 "last_posted_at": "2024-11-16T17:48:29.488Z",339 "bumped": true,340 "bumped_at": "2024-11-16T17:48:29.488Z",341 "archetype": "regular",342 "unseen": false,343 "pinned": false,344 "unpinned": null,345 "visible": true,346 "closed": false,347 "archived": false,348 "bookmarked": null,349 "liked": null,350 "tags_descriptions": {},351 "like_count": 0,352 "views": 690,353 "category_id": 39,354 "featured_link": null,355 "has_accepted_answer": false,356 "posters": [357 {358 "extras": null,359 "description": "Original Poster",360 "user": {361 "id": 8321,362 "username": "will-rice",363 "name": "Will Rice",364 "avatar_template": "/user_avatar/discuss.pytorch.org/will-rice/{size}/55325_2.png",365 "trust_level": 1366 }367 },368 {369 "extras": "latest",370 "description": "Most Recent Poster",371 "user": {372 "id": 41997,373 "username": "bdhirsh",374 "name": "Brian Hirsh",375 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png",376 "trust_level": 2377 }378 }379 ]380 },381 {382 "fancy_title": "What is the uplimit of dynamo graph cache?",383 "id": 215145,384 "title": "What is the uplimit of dynamo graph cache?",385 "slug": "what-is-the-uplimit-of-dynamo-graph-cache",386 "posts_count": 3,387 "reply_count": 1,388 "highest_post_number": 3,389 "image_url": null,390 "created_at": "2025-01-09T02:26:13.918Z",391 "last_posted_at": "2025-01-10T03:16:07.070Z",392 "bumped": true,393 "bumped_at": "2025-01-10T03:16:07.070Z",394 "archetype": "regular",395 "unseen": false,396 "pinned": false,397 "unpinned": null,398 "visible": true,399 "closed": false,400 "archived": false,401 "bookmarked": null,402 "liked": null,403 "tags_descriptions": {},404 "like_count": 0,405 "views": 224,406 "category_id": 39,407 "featured_link": null,408 "has_accepted_answer": true,409 "posters": [410 {411 "extras": null,412 "description": "Original Poster",413 "user": {414 "id": 78373,415 "username": "GodHforever",416 "name": "God Hforever",417 "avatar_template": "/user_avatar/discuss.pytorch.org/godhforever/{size}/72261_2.png",418 "trust_level": 1419 }420 },421 {422 "extras": "latest",423 "description": "Most Recent Poster, Accepted Answer",424 "user": {425 "id": 41997,426 "username": "bdhirsh",427 "name": "Brian Hirsh",428 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png",429 "trust_level": 2430 }431 }432 ]433 },434 {435 "fancy_title": "Decomposition to Aten IR",436 "id": 216033,437 "title": "Decomposition to Aten IR",438 "slug": "decomposition-to-aten-ir",439 "posts_count": 7,440 "reply_count": 5,441 "highest_post_number": 7,442 "image_url": null,443 "created_at": "2025-01-29T20:08:06.353Z",444 "last_posted_at": "2025-01-31T23:45:25.638Z",445 "bumped": true,446 "bumped_at": "2025-01-31T23:45:25.638Z",447 "archetype": "regular",448 "unseen": false,449 "pinned": false,450 "unpinned": null,451 "visible": true,452 "closed": false,453 "archived": false,454 "bookmarked": null,455 "liked": null,456 "tags_descriptions": {},457 "like_count": 2,458 "views": 859,459 "category_id": 39,460 "featured_link": null,461 "has_accepted_answer": true,462 "posters": [463 {464 "extras": "latest",465 "description": "Original Poster, Most Recent Poster",466 "user": {467 "id": 18063,468 "username": "Jerome_Ku",469 "name": "Jerome Ku",470 "avatar_template": "/user_avatar/discuss.pytorch.org/jerome_ku/{size}/11249_2.png",471 "trust_level": 1472 }473 },474 {475 "extras": null,476 "description": "Frequent Poster, Accepted Answer",477 "user": {478 "id": 41997,479 "username": "bdhirsh",480 "name": "Brian Hirsh",481 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png",482 "trust_level": 2483 }484 }485 ]486 },487 {488 "fancy_title": "Does dynamo trigger real kernel execution?",489 "id": 214401,490 "title": "Does dynamo trigger real kernel execution?",491 "slug": "does-dynamo-trigger-real-kernel-execution",492 "posts_count": 4,493 "reply_count": 0,494 "highest_post_number": 4,495 "image_url": null,496 "created_at": "2024-12-19T12:39:27.870Z",497 "last_posted_at": "2025-02-08T06:30:19.098Z",498 "bumped": true,499 "bumped_at": "2025-02-08T06:30:19.098Z",500 "archetype": "regular",501 "unseen": false,502 "pinned": false,503 "unpinned": null,504 "visible": true,505 "closed": false,506 "archived": false,507 "bookmarked": null,508 "liked": null,509 "tags_descriptions": {},510 "like_count": 0,511 "views": 203,512 "category_id": 39,513 "featured_link": null,514 "has_accepted_answer": false,515 "posters": [516 {517 "extras": "latest",518 "description": "Original Poster, Most Recent Poster",519 "user": {520 "id": 49372,521 "username": "yjguo",522 "name": "",523 "avatar_template": "/letter_avatar_proxy/v4/letter/y/db5fbb/{size}.png",524 "trust_level": 1525 }526 },527 {528 "extras": null,529 "description": "Frequent Poster",530 "user": {531 "id": 3534,532 "username": "ptrblck",533 "name": "",534 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",535 "admin": true,536 "moderator": true,537 "trust_level": 2538 }539 }540 ]541 },542 {543 "fancy_title": "Better understanding why AOTAutograd decomposes `fused_rms_norm_backward` for CUDA, but not for Meta tensors",544 "id": 223096,545 "title": "Better understanding why AOTAutograd decomposes `fused_rms_norm_backward` for CUDA, but not for Meta tensors",546 "slug": "better-understanding-why-aotautograd-decomposes-fused-rms-norm-backward-for-cuda-but-not-for-meta-tensors",547 "posts_count": 1,548 "reply_count": 0,549 "highest_post_number": 1,550 "image_url": null,551 "created_at": "2025-09-15T05:54:49.584Z",552 "last_posted_at": "2025-09-15T05:54:49.649Z",553 "bumped": true,554 "bumped_at": "2025-09-15T05:54:49.649Z",555 "archetype": "regular",556 "unseen": false,557 "pinned": false,558 "unpinned": null,559 "visible": true,560 "closed": false,561 "archived": false,562 "bookmarked": null,563 "liked": null,564 "tags_descriptions": {},565 "like_count": 0,566 "views": 20,567 "category_id": 39,568 "featured_link": null,569 "has_accepted_answer": false,570 "posters": [571 {572 "extras": "latest single",573 "description": "Original Poster, Most Recent Poster",574 "user": {575 "id": 85865,576 "username": "jinsun-yoo",577 "name": "Jinsun Yoo",578 "avatar_template": "/user_avatar/discuss.pytorch.org/jinsun-yoo/{size}/78229_2.png",579 "trust_level": 1580 }581 }582 ]583 }584 ],585 "tags_descriptions": {},586 "fancy_title": "Compiling a model that uses sdp_kernel to enable the backends does not work",587 "id": 214200,588 "title": "Compiling a model that uses sdp_kernel to enable the backends does not work",589 "posts_count": 5,590 "created_at": "2024-12-13T14:23:27.650Z",591 "views": 281,592 "reply_count": 2,593 "like_count": 2,594 "last_posted_at": "2024-12-18T20:23:48.812Z",595 "visible": true,596 "closed": false,597 "archived": false,598 "has_summary": false,599 "archetype": "regular",600 "slug": "compiling-a-model-that-uses-sdp-kernel-to-enable-the-backends-does-not-work",601 "category_id": 39,602 "word_count": 335,603 "deleted_at": null,604 "user_id": 75496,605 "featured_link": null,606 "pinned_globally": false,607 "pinned_at": null,608 "pinned_until": null,609 "image_url": null,610 "slow_mode_seconds": 0,611 "draft": null,612 "draft_key": "topic_214200",613 "draft_sequence": null,614 "unpinned": null,615 "pinned": false,616 "current_post_number": 1,617 "highest_post_number": 5,618 "deleted_by": null,619 "actions_summary": [620 {621 "id": 4,622 "count": 0,623 "hidden": false,624 "can_act": false625 },626 {627 "id": 8,628 "count": 0,629 "hidden": false,630 "can_act": false631 },632 {633 "id": 10,634 "count": 0,635 "hidden": false,636 "can_act": false637 },638 {639 "id": 7,640 "count": 0,641 "hidden": false,642 "can_act": false643 }644 ],645 "chunk_size": 20,646 "bookmarked": false,647 "topic_timer": null,648 "message_bus_last_id": 0,649 "participant_count": 3,650 "show_read_indicator": false,651 "thumbnails": null,652 "slow_mode_enabled_until": null,653 "can_vote": false,654 "vote_count": 0,655 "user_voted": false,656 "discourse_zendesk_plugin_zendesk_id": null,657 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",658 "details": {659 "can_edit": false,660 "notification_level": 1,661 "participants": [662 {663 "id": 75496,664 "username": "raulc",665 "name": "Raul Ciotescu",666 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png",667 "post_count": 3,668 "primary_group_name": null,669 "flair_name": null,670 "flair_url": null,671 "flair_color": null,672 "flair_bg_color": null,673 "flair_group_id": null,674 "trust_level": 1675 },676 {677 "id": 3534,678 "username": "ptrblck",679 "name": "",680 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",681 "post_count": 1,682 "primary_group_name": null,683 "flair_name": null,684 "flair_url": null,685 "flair_color": null,686 "flair_bg_color": null,687 "flair_group_id": null,688 "admin": true,689 "moderator": true,690 "trust_level": 2691 },692 {693 "id": 41997,694 "username": "bdhirsh",695 "name": "Brian Hirsh",696 "avatar_template": "/user_avatar/discuss.pytorch.org/bdhirsh/{size}/74038_2.png",697 "post_count": 1,698 "primary_group_name": null,699 "flair_name": null,700 "flair_url": null,701 "flair_color": null,702 "flair_bg_color": null,703 "flair_group_id": null,704 "trust_level": 2705 }706 ],707 "created_by": {708 "id": 75496,709 "username": "raulc",710 "name": "Raul Ciotescu",711 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png"712 },713 "last_poster": {714 "id": 75496,715 "username": "raulc",716 "name": "Raul Ciotescu",717 "avatar_template": "/user_avatar/discuss.pytorch.org/raulc/{size}/69719_2.png"718 }719 },720 "bookmarks": []721 },722 {723 "post_stream": {724 "posts": [725 {726 "id": 461442,727 "name": "forestbat",728 "username": "forestbat",729 "avatar_template": "/user_avatar/discuss.pytorch.org/forestbat/{size}/74440_2.png",730 "created_at": "2024-12-14T17:55:33.326Z",731 "cooked": "<p>The GPU0 in my server has been occupied by others’ processes, so I blocked GPU0 and use <code>mp.spawn</code> to train my model, but it failed to create train process.</p>\n<p>This is reproducible example:</p>\n<pre><code class=\"lang-auto\">import torch\nimport torch.multiprocessing as mp\nimport torch.nn as nn\nimport torch.optim as optim\nimport os\nimport torch.distributed as dist\n\ndef train(rank, gpu_ids):\n device_id = gpu_ids[rank]\n dist.init_process_group(\n backend=\"nccl\", init_method=\"env://\", world_size=len(gpu_ids), rank=device_id)\n torch.cuda.set_device(device_id)\n print(f\"Rank {rank} is using device {device_id}\")\n model = nn.Linear(10, 1).to(device_id)\n optimizer = optim.SGD(model.parameters(), lr=0.01)\n input_data = torch.randn(5, 10).to(device_id)\n target = torch.randn(5, 1).to(device_id)\n for epoch in range(5):\n optimizer.zero_grad()\n output = model(input_data)\n loss = nn.MSELoss()(output, target)\n loss.backward()\n optimizer.step()\n print(f\"Rank {rank}, Epoch {epoch}, Loss: {loss.item()}\")\n\ndef test_main():\n os.environ['CUDA_VISIBLE_DEVICES'] = '1,2'\n os.environ[\"MASTER_ADDR\"] = '127.0.0.1'\n os.environ[\"MASTER_PORT\"] = '12345'\n gpu_ids = [1]\n world_size = len(gpu_ids)\n mp.spawn(train, args=(gpu_ids,), nprocs=world_size, join=True)\n</code></pre>\n<p>In my test, the <code>train()</code> method stucked in here:</p>\n<pre><code class=\"lang-auto\">dist.init_process_group(backend=\"nccl\", init_method=\"env://\", world_size=len(gpu_ids), rank=device_id)\n</code></pre>\n<p>So what happened and how to solve it?</p>",732 "post_number": 1,733 "post_type": 1,734 "posts_count": 2,735 "updated_at": "2024-12-14T17:55:33.326Z",736 "reply_count": 0,737 "reply_to_post_number": null,738 "quote_count": 0,739 "incoming_link_count": 10,740 "reads": 11,741 "readers_count": 10,742 "score": 52.2,743 "yours": false,744 "topic_id": 214228,745 "topic_slug": "pytorch-start-training-process-stucked-when-blocking-gpu-0",746 "display_username": "forestbat",747 "primary_group_name": null,748 "flair_name": null,749 "flair_url": null,750 "flair_bg_color": null,751 "flair_color": null,752 "flair_group_id": null,753 "badges_granted": [],754 "version": 1,755 "can_edit": false,756 "can_delete": false,757 "can_recover": false,758 "can_see_hidden_post": false,759 "can_wiki": false,760 "read": true,761 "user_title": null,762 "bookmarked": false,763 "actions_summary": [],764 "moderator": false,765 "admin": false,766 "staff": false,767 "user_id": 81406,768 "hidden": false,769 "trust_level": 1,770 "deleted_at": null,771 "user_deleted": false,772 "edit_reason": null,773 "can_view_edit_history": true,774 "wiki": false,775 "post_url": "/t/pytorch-start-training-process-stucked-when-blocking-gpu-0/214228/1",776 "can_accept_answer": false,777 "can_unaccept_answer": false,778 "accepted_answer": false,779 "topic_accepted_answer": null,780 "can_vote": false781 },782 {783 "id": 461692,784 "name": "Matthias",785 "username": "mreso",786 "avatar_template": "/letter_avatar_proxy/v4/letter/m/41988e/{size}.png",787 "created_at": "2024-12-18T18:04:11.216Z",788 "cooked": "<p>Hi <a class=\"mention\" href=\"/u/forestbat\">@forestbat</a>,</p>\n<p>this is because you’re giving device_id as rank to init_process_group which is 1 in your case. And as world size is 1 in your case (len(gpu_ids)) it probably waits for rank 0 to show up.</p>\n<p>If you switch to rank=rank instead of device_id it works but there is another issue. As soon as you define CUDA_VISIBLE_DEVICES you can just address the GPUs with 0,1,2,… etc. The mapping from cuda:0 to gpu1 happens automatic. In your code your model will actually end up on gpu2 as cuda:1 maps to gpu2.</p>",789 "post_number": 2,790 "post_type": 1,791 "posts_count": 2,792 "updated_at": "2024-12-18T18:04:11.216Z",793 "reply_count": 0,794 "reply_to_post_number": null,795 "quote_count": 0,796 "incoming_link_count": 2,797 "reads": 7,798 "readers_count": 6,799 "score": 11.4,800 "yours": false,801 "topic_id": 214228,802 "topic_slug": "pytorch-start-training-process-stucked-when-blocking-gpu-0",803 "display_username": "Matthias",804 "primary_group_name": null,805 "flair_name": null,806 "flair_url": null,807 "flair_bg_color": null,808 "flair_color": null,809 "flair_group_id": null,810 "badges_granted": [],811 "version": 1,812 "can_edit": false,813 "can_delete": false,814 "can_recover": false,815 "can_see_hidden_post": false,816 "can_wiki": false,817 "read": true,818 "user_title": null,819 "bookmarked": false,820 "actions_summary": [],821 "moderator": false,822 "admin": false,823 "staff": false,824 "user_id": 62821,825 "hidden": false,826 "trust_level": 1,827 "deleted_at": null,828 "user_deleted": false,829 "edit_reason": null,830 "can_view_edit_history": true,831 "wiki": false,832 "post_url": "/t/pytorch-start-training-process-stucked-when-blocking-gpu-0/214228/2",833 "can_accept_answer": false,834 "can_unaccept_answer": false,835 "accepted_answer": false,836 "topic_accepted_answer": null837 }838 ],839 "stream": [840 461442,841 461692842 ]843 },844 "timeline_lookup": [845 [846 1,847 315848 ],849 [850 2,851 311852 ]853 ],854 "suggested_topics": [855 {856 "fancy_title": "How to handle training of few layers with DDP",857 "id": 219979,858 "title": "How to handle training of few layers with DDP",859 "slug": "how-to-handle-training-of-few-layers-with-ddp",860 "posts_count": 3,861 "reply_count": 1,862 "highest_post_number": 3,863 "image_url": null,864 "created_at": "2025-05-13T08:17:42.196Z",865 "last_posted_at": "2025-05-14T08:40:41.263Z",866 "bumped": true,867 "bumped_at": "2025-05-14T08:40:41.263Z",868 "archetype": "regular",869 "unseen": false,870 "pinned": false,871 "unpinned": null,872 "visible": true,873 "closed": false,874 "archived": false,875 "bookmarked": null,876 "liked": null,877 "tags_descriptions": {},878 "like_count": 0,879 "views": 62,880 "category_id": 12,881 "featured_link": null,882 "has_accepted_answer": true,883 "posters": [884 {885 "extras": "latest",886 "description": "Original Poster, Most Recent Poster",887 "user": {888 "id": 21407,889 "username": "Guillaume_Jeanneret",890 "name": "Guillaume Jeanneret",891 "avatar_template": "/user_avatar/discuss.pytorch.org/guillaume_jeanneret/{size}/14837_2.png",892 "trust_level": 1893 }894 },895 {896 "extras": null,897 "description": "Frequent Poster, Accepted Answer",898 "user": {899 "id": 3534,900 "username": "ptrblck",901 "name": "",902 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",903 "admin": true,904 "moderator": true,905 "trust_level": 2906 }907 }908 ]909 },910 {911 "fancy_title": "Using Queue in multi GPU training",912 "id": 214634,913 "title": "Using Queue in multi GPU training",914 "slug": "using-queue-in-multi-gpu-training",915 "posts_count": 5,916 "reply_count": 3,917 "highest_post_number": 5,918 "image_url": null,919 "created_at": "2024-12-25T19:07:46.693Z",920 "last_posted_at": "2024-12-30T23:11:30.761Z",921 "bumped": true,922 "bumped_at": "2024-12-30T23:11:30.761Z",923 "archetype": "regular",924 "unseen": false,925 "pinned": false,926 "unpinned": null,927 "visible": true,928 "closed": false,929 "archived": false,930 "bookmarked": null,931 "liked": null,932 "tags_descriptions": {},933 "like_count": 0,934 "views": 446,935 "category_id": 12,936 "featured_link": null,937 "has_accepted_answer": true,938 "posters": [939 {940 "extras": "latest",941 "description": "Original Poster, Most Recent Poster",942 "user": {943 "id": 48236,944 "username": "jshcht",945 "name": "haozhi",946 "avatar_template": "/user_avatar/discuss.pytorch.org/jshcht/{size}/41390_2.png",947 "trust_level": 1948 }949 },950 {951 "extras": null,952 "description": "Frequent Poster, Accepted Answer",953 "user": {954 "id": 39542,955 "username": "H-Huang",956 "name": "Howard Huang",957 "avatar_template": "/user_avatar/discuss.pytorch.org/h-huang/{size}/35598_2.png",958 "trust_level": 2959 }960 }961 ]962 },963 {964 "fancy_title": "Gather data from multiple processes in one gpu",965 "id": 213192,966 "title": "Gather data from multiple processes in one gpu",967 "slug": "gather-data-from-multiple-processes-in-one-gpu",968 "posts_count": 3,969 "reply_count": 0,970 "highest_post_number": 3,971 "image_url": null,972 "created_at": "2024-11-20T09:17:23.525Z",973 "last_posted_at": "2024-11-22T20:26:13.425Z",974 "bumped": true,975 "bumped_at": "2024-11-22T20:26:13.425Z",976 "archetype": "regular",977 "unseen": false,978 "pinned": false,979 "unpinned": null,980 "visible": true,981 "closed": false,982 "archived": false,983 "bookmarked": null,984 "liked": null,985 "tags_descriptions": {},986 "like_count": 1,987 "views": 439,988 "category_id": 12,989 "featured_link": null,990 "has_accepted_answer": false,991 "posters": [992 {993 "extras": null,994 "description": "Original Poster",995 "user": {996 "id": 81018,997 "username": "Huang_Jiawei",998 "name": "Huang Jiawei",999 "avatar_template": "/user_avatar/discuss.pytorch.org/huang_jiawei/{size}/73664_2.png",1000 "trust_level": 01001 }1002 },1003 {1004 "extras": null,1005 "description": "Frequent Poster",1006 "user": {1007 "id": 78505,1008 "username": "tianyu",1009 "name": "",1010 "avatar_template": "/user_avatar/discuss.pytorch.org/tianyu/{size}/72368_2.png",1011 "trust_level": 21012 }1013 },1014 {1015 "extras": "latest",1016 "description": "Most Recent Poster",1017 "user": {1018 "id": 55215,1019 "username": "kwen2501",1020 "name": "Ke Wen",1021 "avatar_template": "/user_avatar/discuss.pytorch.org/kwen2501/{size}/48844_2.png",1022 "trust_level": 11023 }1024 }1025 ]1026 },1027 {1028 "fancy_title": "How to perform distributed communication (NCCL) using LibTorch?",1029 "id": 219558,1030 "title": "How to perform distributed communication (NCCL) using LibTorch?",1031 "slug": "how-to-perform-distributed-communication-nccl-using-libtorch",1032 "posts_count": 4,1033 "reply_count": 2,1034 "highest_post_number": 4,1035 "image_url": null,1036 "created_at": "2025-04-28T16:18:26.591Z",1037 "last_posted_at": "2025-05-04T06:01:53.711Z",1038 "bumped": true,1039 "bumped_at": "2025-05-04T06:01:53.711Z",1040 "archetype": "regular",1041 "unseen": false,1042 "pinned": false,1043 "unpinned": null,1044 "visible": true,1045 "closed": false,1046 "archived": false,1047 "bookmarked": null,1048 "liked": null,1049 "tags_descriptions": {},1050 "like_count": 0,1051 "views": 78,1052 "category_id": 12,1053 "featured_link": null,1054 "has_accepted_answer": true,1055 "posters": [1056 {1057 "extras": "latest",1058 "description": "Original Poster, Most Recent Poster",1059 "user": {1060 "id": 84054,1061 "username": "Jiawen_Niu",1062 "name": "Jiawen Niu",1063 "avatar_template": "/user_avatar/discuss.pytorch.org/jiawen_niu/{size}/75032_2.png",1064 "trust_level": 11065 }1066 },1067 {1068 "extras": null,1069 "description": "Frequent Poster, Accepted Answer",1070 "user": {1071 "id": 54320,1072 "username": "fduwjj",1073 "name": "Hugo",1074 "avatar_template": "/user_avatar/discuss.pytorch.org/fduwjj/{size}/47855_2.png",1075 "trust_level": 21076 }1077 }1078 ]1079 },1080 {1081 "fancy_title": "Support for Overlapping AllGather and ReduceScatter in FSDP",1082 "id": 222489,1083 "title": "Support for Overlapping AllGather and ReduceScatter in FSDP",1084 "slug": "support-for-overlapping-allgather-and-reducescatter-in-fsdp",1085 "posts_count": 3,1086 "reply_count": 1,1087 "highest_post_number": 3,1088 "image_url": null,1089 "created_at": "2025-08-20T00:20:21.400Z",1090 "last_posted_at": "2025-08-25T04:38:02.568Z",1091 "bumped": true,1092 "bumped_at": "2025-08-25T04:38:02.568Z",1093 "archetype": "regular",1094 "unseen": false,1095 "pinned": false,1096 "unpinned": null,1097 "visible": true,1098 "closed": false,1099 "archived": false,1100 "bookmarked": null,1101 "liked": null,1102 "tags_descriptions": {},1103 "like_count": 0,1104 "views": 62,1105 "category_id": 12,1106 "featured_link": null,1107 "has_accepted_answer": false,1108 "posters": [1109 {1110 "extras": "latest",1111 "description": "Original Poster, Most Recent Poster",1112 "user": {1113 "id": 81938,1114 "username": "nariaki3551",1115 "name": "Nariaki Tateiwa",1116 "avatar_template": "/user_avatar/discuss.pytorch.org/nariaki3551/{size}/74961_2.png",1117 "trust_level": 11118 }1119 },1120 {1121 "extras": null,1122 "description": "Frequent Poster",1123 "user": {1124 "id": 3534,1125 "username": "ptrblck",1126 "name": "",1127 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1128 "admin": true,1129 "moderator": true,1130 "trust_level": 21131 }1132 }1133 ]1134 }1135 ],1136 "tags_descriptions": {},1137 "fancy_title": "Pytorch start training process stucked when blocking GPU 0",1138 "id": 214228,1139 "title": "Pytorch start training process stucked when blocking GPU 0",1140 "posts_count": 2,1141 "created_at": "2024-12-14T17:55:33.286Z",1142 "views": 204,1143 "reply_count": 0,1144 "like_count": 0,1145 "last_posted_at": "2024-12-18T18:04:11.216Z",1146 "visible": true,1147 "closed": false,1148 "archived": false,1149 "has_summary": false,1150 "archetype": "regular",1151 "slug": "pytorch-start-training-process-stucked-when-blocking-gpu-0",1152 "category_id": 12,1153 "word_count": 303,1154 "deleted_at": null,1155 "user_id": 81406,1156 "featured_link": null,1157 "pinned_globally": false,1158 "pinned_at": null,1159 "pinned_until": null,1160 "image_url": null,1161 "slow_mode_seconds": 0,1162 "draft": null,1163 "draft_key": "topic_214228",1164 "draft_sequence": null,1165 "unpinned": null,1166 "pinned": false,1167 "current_post_number": 1,1168 "highest_post_number": 2,1169 "deleted_by": null,1170 "actions_summary": [1171 {1172 "id": 4,1173 "count": 0,1174 "hidden": false,1175 "can_act": false1176 },1177 {1178 "id": 8,1179 "count": 0,1180 "hidden": false,1181 "can_act": false1182 },1183 {1184 "id": 10,1185 "count": 0,1186 "hidden": false,1187 "can_act": false1188 },1189 {1190 "id": 7,1191 "count": 0,1192 "hidden": false,1193 "can_act": false1194 }1195 ],1196 "chunk_size": 20,1197 "bookmarked": false,1198 "topic_timer": null,1199 "message_bus_last_id": 0,1200 "participant_count": 2,