Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 313825,7 "name": "nikiguo",8 "username": "nikiguo93",9 "avatar_template": "/user_avatar/discuss.pytorch.org/nikiguo93/{size}/30791_2.png",10 "created_at": "2021-10-27T11:54:29.534Z",11 "cooked": "<p>Hi all. I want to create a linear layer and then initialize it with specific weights(I have a .sav file). Can anybody tell me how to realize it with pytorch?<br>\nafter doing this I should discard some layers of pretrained alexnet and add this linear layer to the alexnet. Can anybody tell me how to realize that with pytorch?</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 4,15 "updated_at": "2021-10-27T11:54:29.534Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 1121,20 "reads": 16,21 "readers_count": 15,22 "score": 5603.2,23 "yours": false,24 "topic_id": 135239,25 "topic_slug": "how-to-create-a-linear-layer-and-initialize-it-with-specific-weight",26 "display_username": "nikiguo",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 44418,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/how-to-create-a-linear-layer-and-initialize-it-with-specific-weight/135239/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 313886,64 "name": "",65 "username": "ptrblck",66 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",67 "created_at": "2021-10-27T19:22:07.981Z",68 "cooked": "<p>You can load parameters via:</p>\n<pre><code class=\"lang-python\">with torch.no_grad():\n model.layer.weight.copy_(custom_weight_tensor)\n</code></pre>\n<p>I don’t know exactly what “discard” layers means, but in case you want to remove them you could write a custom model and reuse other layers or replace them with <code>nn.Identity</code> assuming that the shapes of the activations would still match.</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 4,72 "updated_at": "2021-10-27T19:22:07.981Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 8,77 "reads": 16,78 "readers_count": 15,79 "score": 48.2,80 "yours": false,81 "topic_id": 135239,82 "topic_slug": "how-to-create-a-linear-layer-and-initialize-it-with-specific-weight",83 "display_username": "",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 3534,105 "hidden": false,106 "trust_level": 2,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/how-to-create-a-linear-layer-and-initialize-it-with-specific-weight/135239/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 315071,120 "name": "nikiguo",121 "username": "nikiguo93",122 "avatar_template": "/user_avatar/discuss.pytorch.org/nikiguo93/{size}/30791_2.png",123 "created_at": "2021-11-03T11:21:34.525Z",124 "cooked": "<p>Hi, thank you for your information. I trained a linear regression model and now I want to use these weights and bias from linear regression model to initialize the linear layer. I hope that the linear layer can behave like the linear regression model. how to realize that?</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 4,128 "updated_at": "2021-11-03T11:21:34.525Z",129 "reply_count": 1,130 "reply_to_post_number": 2,131 "quote_count": 0,132 "incoming_link_count": 2,133 "reads": 14,134 "readers_count": 13,135 "score": 17.8,136 "yours": false,137 "topic_id": 135239,138 "topic_slug": "how-to-create-a-linear-layer-and-initialize-it-with-specific-weight",139 "display_username": "nikiguo",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 1,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": null,155 "reply_to_user": {156 "id": 3534,157 "username": "ptrblck",158 "name": "",159 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"160 },161 "bookmarked": false,162 "actions_summary": [],163 "moderator": false,164 "admin": false,165 "staff": false,166 "user_id": 44418,167 "hidden": false,168 "trust_level": 1,169 "deleted_at": null,170 "user_deleted": false,171 "edit_reason": null,172 "can_view_edit_history": true,173 "wiki": false,174 "post_url": "/t/how-to-create-a-linear-layer-and-initialize-it-with-specific-weight/135239/3",175 "can_accept_answer": false,176 "can_unaccept_answer": false,177 "accepted_answer": false,178 "topic_accepted_answer": null179 },180 {181 "id": 315193,182 "name": "",183 "username": "ptrblck",184 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",185 "created_at": "2021-11-03T18:30:59.494Z",186 "cooked": "<p>I don’t know how the architecture of the linear regression model looks like, but in case it’s using non-linear activation functions you wouldn’t be able to squeeze this model into a single linear layer.<br>\nOtherwise I would assume that training a single linear layer would also work for your use case.</p>",187 "post_number": 4,188 "post_type": 1,189 "posts_count": 4,190 "updated_at": "2021-11-03T18:30:59.494Z",191 "reply_count": 0,192 "reply_to_post_number": 3,193 "quote_count": 0,194 "incoming_link_count": 3,195 "reads": 12,196 "readers_count": 11,197 "score": 17.4,198 "yours": false,199 "topic_id": 135239,200 "topic_slug": "how-to-create-a-linear-layer-and-initialize-it-with-specific-weight",201 "display_username": "",202 "primary_group_name": null,203 "flair_name": null,204 "flair_url": null,205 "flair_bg_color": null,206 "flair_color": null,207 "flair_group_id": null,208 "badges_granted": [],209 "version": 1,210 "can_edit": false,211 "can_delete": false,212 "can_recover": false,213 "can_see_hidden_post": false,214 "can_wiki": false,215 "read": true,216 "user_title": "",217 "reply_to_user": {218 "id": 44418,219 "username": "nikiguo93",220 "name": "nikiguo",221 "avatar_template": "/user_avatar/discuss.pytorch.org/nikiguo93/{size}/30791_2.png"222 },223 "bookmarked": false,224 "actions_summary": [],225 "moderator": true,226 "admin": true,227 "staff": true,228 "user_id": 3534,229 "hidden": false,230 "trust_level": 2,231 "deleted_at": null,232 "user_deleted": false,233 "edit_reason": null,234 "can_view_edit_history": true,235 "wiki": false,236 "post_url": "/t/how-to-create-a-linear-layer-and-initialize-it-with-specific-weight/135239/4",237 "can_accept_answer": false,238 "can_unaccept_answer": false,239 "accepted_answer": false,240 "topic_accepted_answer": null241 }242 ],243 "stream": [244 313825,245 313886,246 315071,247 315193248 ]249 },250 "timeline_lookup": [251 [252 1,253 1459254 ],255 [256 3,257 1452258 ]259 ],260 "suggested_topics": [261 {262 "fancy_title": "Ai model on pytorch",263 "id": 217579,264 "title": "Ai model on pytorch",265 "slug": "ai-model-on-pytorch",266 "posts_count": 4,267 "reply_count": 2,268 "highest_post_number": 4,269 "image_url": null,270 "created_at": "2025-03-08T04:59:06.797Z",271 "last_posted_at": "2025-03-08T12:13:16.534Z",272 "bumped": true,273 "bumped_at": "2025-03-08T12:13:16.534Z",274 "archetype": "regular",275 "unseen": false,276 "pinned": false,277 "unpinned": null,278 "visible": true,279 "closed": false,280 "archived": false,281 "bookmarked": null,282 "liked": null,283 "tags_descriptions": {},284 "like_count": 0,285 "views": 95,286 "category_id": 1,287 "featured_link": null,288 "has_accepted_answer": false,289 "posters": [290 {291 "extras": null,292 "description": "Original Poster",293 "user": {294 "id": 83132,295 "username": "iamsemka_41",296 "name": "iamsemka 41",297 "avatar_template": "/user_avatar/discuss.pytorch.org/iamsemka_41/{size}/76034_2.png",298 "trust_level": 0299 }300 },301 {302 "extras": "latest",303 "description": "Most Recent Poster",304 "user": {305 "id": 64488,306 "username": "Naming-isDifficult",307 "name": "",308 "avatar_template": "/user_avatar/discuss.pytorch.org/naming-isdifficult/{size}/58644_2.png",309 "trust_level": 2310 }311 }312 ]313 },314 {315 "fancy_title": "Matrix multiplication implementation in PyTorch",316 "id": 214503,317 "title": "Matrix multiplication implementation in PyTorch",318 "slug": "matrix-multiplication-implementation-in-pytorch",319 "posts_count": 5,320 "reply_count": 3,321 "highest_post_number": 5,322 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/1/1/1164c0316f8a3e99f93bb18951f2174bd5a5b9cd_2_1024x920.jpeg",323 "created_at": "2024-12-21T14:51:31.856Z",324 "last_posted_at": "2025-01-02T17:24:33.016Z",325 "bumped": true,326 "bumped_at": "2025-01-02T17:24:33.016Z",327 "archetype": "regular",328 "unseen": false,329 "pinned": false,330 "unpinned": null,331 "visible": true,332 "closed": false,333 "archived": false,334 "bookmarked": null,335 "liked": null,336 "tags_descriptions": {},337 "like_count": 2,338 "views": 890,339 "category_id": 1,340 "featured_link": null,341 "has_accepted_answer": true,342 "posters": [343 {344 "extras": null,345 "description": "Original Poster",346 "user": {347 "id": 81557,348 "username": "Areg_Petrosyan",349 "name": "Areg Petrosyan",350 "avatar_template": "/user_avatar/discuss.pytorch.org/areg_petrosyan/{size}/74662_2.png",351 "trust_level": 1352 }353 },354 {355 "extras": null,356 "description": "Frequent Poster, Accepted Answer",357 "user": {358 "id": 211,359 "username": "albanD",360 "name": "Alban D",361 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",362 "admin": true,363 "moderator": true,364 "trust_level": 4365 }366 },367 {368 "extras": null,369 "description": "Frequent Poster",370 "user": {371 "id": 2544,372 "username": "blackberry",373 "name": "Blackberry",374 "avatar_template": "/letter_avatar_proxy/v4/letter/b/ecd19e/{size}.png",375 "trust_level": 2376 }377 },378 {379 "extras": "latest",380 "description": "Most Recent Poster",381 "user": {382 "id": 78368,383 "username": "swolchok",384 "name": "Scott Wolchok",385 "avatar_template": "/user_avatar/discuss.pytorch.org/swolchok/{size}/72256_2.png",386 "trust_level": 3387 }388 }389 ]390 },391 {392 "fancy_title": "Wrong nvcc version when compiling plugins",393 "id": 216293,394 "title": "Wrong nvcc version when compiling plugins",395 "slug": "wrong-nvcc-version-when-compiling-plugins",396 "posts_count": 1,397 "reply_count": 0,398 "highest_post_number": 1,399 "image_url": null,400 "created_at": "2025-02-06T01:00:59.715Z",401 "last_posted_at": "2025-02-06T01:00:59.757Z",402 "bumped": true,403 "bumped_at": "2025-02-06T01:02:58.568Z",404 "archetype": "regular",405 "unseen": false,406 "pinned": false,407 "unpinned": null,408 "visible": true,409 "closed": false,410 "archived": false,411 "bookmarked": null,412 "liked": null,413 "tags_descriptions": {},414 "like_count": 0,415 "views": 82,416 "category_id": 1,417 "featured_link": null,418 "has_accepted_answer": false,419 "posters": [420 {421 "extras": "latest single",422 "description": "Original Poster, Most Recent Poster",423 "user": {424 "id": 29433,425 "username": "cltexe",426 "name": "Omer Faruk Soylemez",427 "avatar_template": "/user_avatar/discuss.pytorch.org/cltexe/{size}/41817_2.png",428 "trust_level": 1429 }430 }431 ]432 },433 {434 "fancy_title": "Inference with float16",435 "id": 216695,436 "title": "Inference with float16",437 "slug": "inference-with-float16",438 "posts_count": 3,439 "reply_count": 0,440 "highest_post_number": 3,441 "image_url": null,442 "created_at": "2025-02-14T21:52:59.904Z",443 "last_posted_at": "2025-02-18T14:43:09.966Z",444 "bumped": true,445 "bumped_at": "2025-02-18T14:43:09.966Z",446 "archetype": "regular",447 "unseen": false,448 "pinned": false,449 "unpinned": null,450 "visible": true,451 "closed": false,452 "archived": false,453 "bookmarked": null,454 "liked": null,455 "tags_descriptions": {},456 "like_count": 0,457 "views": 206,458 "category_id": 1,459 "featured_link": null,460 "has_accepted_answer": true,461 "posters": [462 {463 "extras": "latest",464 "description": "Original Poster, Most Recent Poster",465 "user": {466 "id": 82696,467 "username": "ytesfai",468 "name": "Yohannes",469 "avatar_template": "/letter_avatar_proxy/v4/letter/y/9fc29f/{size}.png",470 "trust_level": 0471 }472 },473 {474 "extras": null,475 "description": "Frequent Poster, Accepted Answer",476 "user": {477 "id": 3534,478 "username": "ptrblck",479 "name": "",480 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",481 "admin": true,482 "moderator": true,483 "trust_level": 2484 }485 }486 ]487 },488 {489 "fancy_title": "Avoid caching for parametrization",490 "id": 217737,491 "title": "Avoid caching for parametrization",492 "slug": "avoid-caching-for-parametrization",493 "posts_count": 1,494 "reply_count": 0,495 "highest_post_number": 1,496 "image_url": null,497 "created_at": "2025-03-12T09:48:21.626Z",498 "last_posted_at": "2025-03-12T09:48:21.666Z",499 "bumped": true,500 "bumped_at": "2025-03-12T09:48:21.666Z",501 "archetype": "regular",502 "unseen": false,503 "pinned": false,504 "unpinned": null,505 "visible": true,506 "closed": false,507 "archived": false,508 "bookmarked": null,509 "liked": null,510 "tags_descriptions": {},511 "like_count": 0,512 "views": 52,513 "category_id": 1,514 "featured_link": null,515 "has_accepted_answer": false,516 "posters": [517 {518 "extras": "latest single",519 "description": "Original Poster, Most Recent Poster",520 "user": {521 "id": 83213,522 "username": "Franckma",523 "name": "",524 "avatar_template": "/letter_avatar_proxy/v4/letter/f/ccd318/{size}.png",525 "trust_level": 1526 }527 }528 ]529 }530 ],531 "tags_descriptions": {},532 "fancy_title": "How to create a linear layer and initialize it with specific weight?",533 "id": 135239,534 "title": "How to create a linear layer and initialize it with specific weight?",535 "posts_count": 4,536 "created_at": "2021-10-27T11:54:29.468Z",537 "views": 1711,538 "reply_count": 2,539 "like_count": 0,540 "last_posted_at": "2021-11-03T18:30:59.494Z",541 "visible": true,542 "closed": false,543 "archived": false,544 "has_summary": false,545 "archetype": "regular",546 "slug": "how-to-create-a-linear-layer-and-initialize-it-with-specific-weight",547 "category_id": 1,548 "word_count": 220,549 "deleted_at": null,550 "user_id": 44418,551 "featured_link": null,552 "pinned_globally": false,553 "pinned_at": null,554 "pinned_until": null,555 "image_url": null,556 "slow_mode_seconds": 0,557 "draft": null,558 "draft_key": "topic_135239",559 "draft_sequence": null,560 "unpinned": null,561 "pinned": false,562 "current_post_number": 1,563 "highest_post_number": 4,564 "deleted_by": null,565 "actions_summary": [566 {567 "id": 4,568 "count": 0,569 "hidden": false,570 "can_act": false571 },572 {573 "id": 8,574 "count": 0,575 "hidden": false,576 "can_act": false577 },578 {579 "id": 10,580 "count": 0,581 "hidden": false,582 "can_act": false583 },584 {585 "id": 7,586 "count": 0,587 "hidden": false,588 "can_act": false589 }590 ],591 "chunk_size": 20,592 "bookmarked": false,593 "topic_timer": null,594 "message_bus_last_id": 0,595 "participant_count": 2,596 "show_read_indicator": false,597 "thumbnails": null,598 "slow_mode_enabled_until": null,599 "can_vote": false,600 "vote_count": 0,601 "user_voted": false,602 "discourse_zendesk_plugin_zendesk_id": null,603 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",604 "details": {605 "can_edit": false,606 "notification_level": 1,607 "participants": [608 {609 "id": 3534,610 "username": "ptrblck",611 "name": "",612 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",613 "post_count": 2,614 "primary_group_name": null,615 "flair_name": null,616 "flair_url": null,617 "flair_color": null,618 "flair_bg_color": null,619 "flair_group_id": null,620 "admin": true,621 "moderator": true,622 "trust_level": 2623 },624 {625 "id": 44418,626 "username": "nikiguo93",627 "name": "nikiguo",628 "avatar_template": "/user_avatar/discuss.pytorch.org/nikiguo93/{size}/30791_2.png",629 "post_count": 2,630 "primary_group_name": null,631 "flair_name": null,632 "flair_url": null,633 "flair_color": null,634 "flair_bg_color": null,635 "flair_group_id": null,636 "trust_level": 1637 }638 ],639 "created_by": {640 "id": 44418,641 "username": "nikiguo93",642 "name": "nikiguo",643 "avatar_template": "/user_avatar/discuss.pytorch.org/nikiguo93/{size}/30791_2.png"644 },645 "last_poster": {646 "id": 3534,647 "username": "ptrblck",648 "name": "",649 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"650 }651 },652 "bookmarks": []653 },654 {655 "post_stream": {656 "posts": [657 {658 "id": 314802,659 "name": "Suho Cho",660 "username": "thecho7",661 "avatar_template": "/letter_avatar_proxy/v4/letter/t/eada6e/{size}.png",662 "created_at": "2021-11-02T07:55:56.338Z",663 "cooked": "<p>I found out the imbalance of GPU usages on my implementation.</p>\n<pre><code>def train(device, args):\n torch.distributed.init_process_group(backend='nccl', rank=device, world_size=torch.cuda_device_count())\n model_A = A()\n model_B = B()\n \n # Not gonna update A\n model_A.to(device)\n ckpt = torch.load(path)\n model_A.load_state_dict(ckpt['model_state_dict '])\n model_A = torch.nn.parallel.DIstributedDataParallel(model_A, device_ids=[device])\n model_A.eval()\n\n # Gonna update B\n model_B.to(device)\n model_B = torch.nn.parallel.DistributedDataParallel(model_B, device_ids=[device])\n model_B.train() \n\nif __name__ == '__main__':\n args = argparse()\n torch.multiprocessing.spawn(train, nprocs=torch.cuda.device_count(), args=(args, ))\n</code></pre>\n<p>I intend to load both models A and B on GPUs however,<br>\nGPU usage tells me only model_A is allocated on GPU:0 and not allocated on the other GPUs.<br>\nlike below,</p>\n<p>GPU 0: 7000MiB / 11019MiB (model A, B)<br>\nGPU 1: 4000MiB / 11019MiB (model B)<br>\nGPU 2: 4000MiB / 11019MiB (model B)<br>\nGPU 3: 4000MiB / 11019MiB (model B)</p>\n<p>Please ask me free if you have any unclears.</p>",664 "post_number": 1,665 "post_type": 1,666 "posts_count": 4,667 "updated_at": "2021-11-03T00:17:13.492Z",668 "reply_count": 0,669 "reply_to_post_number": null,670 "quote_count": 0,671 "incoming_link_count": 364,672 "reads": 19,673 "readers_count": 18,674 "score": 1818.8,675 "yours": false,676 "topic_id": 135706,677 "topic_slug": "load-multiple-models-with-distributeddataparallel",678 "display_username": "Suho Cho",679 "primary_group_name": null,680 "flair_name": null,681 "flair_url": null,682 "flair_bg_color": null,683 "flair_color": null,684 "flair_group_id": null,685 "badges_granted": [],686 "version": 3,687 "can_edit": false,688 "can_delete": false,689 "can_recover": false,690 "can_see_hidden_post": false,691 "can_wiki": false,692 "read": true,693 "user_title": "",694 "bookmarked": false,695 "actions_summary": [],696 "moderator": false,697 "admin": false,698 "staff": false,699 "user_id": 7263,700 "hidden": false,701 "trust_level": 2,702 "deleted_at": null,703 "user_deleted": false,704 "edit_reason": null,705 "can_view_edit_history": true,706 "wiki": false,707 "post_url": "/t/load-multiple-models-with-distributeddataparallel/135706/1",708 "can_accept_answer": false,709 "can_unaccept_answer": false,710 "accepted_answer": false,711 "topic_accepted_answer": true,712 "can_vote": false713 },714 {715 "id": 314851,716 "name": "Shen Li",717 "username": "mrshenli",718 "avatar_template": "/user_avatar/discuss.pytorch.org/mrshenli/{size}/12220_2.png",719 "created_at": "2021-11-02T14:46:18.531Z",720 "cooked": "<p>It could be all processes unintentionally created CUDA context on the default GPU (<code>cuda:0</code>). To avoid this situation, can you try setting <code>CUDA_VISIBLE_DEVICES</code> env var to a different device for each process, so that each process would only see one GPU?</p>",721 "post_number": 2,722 "post_type": 1,723 "posts_count": 4,724 "updated_at": "2021-11-02T14:46:18.531Z",725 "reply_count": 1,726 "reply_to_post_number": null,727 "quote_count": 0,728 "incoming_link_count": 2,729 "reads": 16,730 "readers_count": 15,731 "score": 18.2,732 "yours": false,733 "topic_id": 135706,734 "topic_slug": "load-multiple-models-with-distributeddataparallel",735 "display_username": "Shen Li",736 "primary_group_name": null,737 "flair_name": null,738 "flair_url": null,739 "flair_bg_color": null,740 "flair_color": null,741 "flair_group_id": null,742 "badges_granted": [],743 "version": 1,744 "can_edit": false,745 "can_delete": false,746 "can_recover": false,747 "can_see_hidden_post": false,748 "can_wiki": false,749 "read": true,750 "user_title": null,751 "bookmarked": false,752 "actions_summary": [],753 "moderator": false,754 "admin": false,755 "staff": false,756 "user_id": 17068,757 "hidden": false,758 "trust_level": 2,759 "deleted_at": null,760 "user_deleted": false,761 "edit_reason": null,762 "can_view_edit_history": true,763 "wiki": false,764 "post_url": "/t/load-multiple-models-with-distributeddataparallel/135706/2",765 "can_accept_answer": false,766 "can_unaccept_answer": false,767 "accepted_answer": false,768 "topic_accepted_answer": true769 },770 {771 "id": 314934,772 "name": "Suho Cho",773 "username": "thecho7",774 "avatar_template": "/letter_avatar_proxy/v4/letter/t/eada6e/{size}.png",775 "created_at": "2021-11-03T00:19:19.675Z",776 "cooked": "<p>I solved this issue by <strong>map_location='cpu’</strong><br>\nLoading pretrained model</p>\n<pre><code>ckpt = torch.load(path)\n</code></pre>\n<p>automatically allocates the parameters to GPU:0.<br>\nI cannot understand why until now, but it is solved by using map_location while I load the model.</p>\n<pre><code>ckpt = torch.load(path, map_location='cpu')\nmodel_A.load_state_dict(ckpt['model_state_dict '])\nmodel_A.to(device)</code></pre>",777 "post_number": 3,778 "post_type": 1,779 "posts_count": 4,780 "updated_at": "2021-11-03T01:26:47.307Z",781 "reply_count": 1,782 "reply_to_post_number": 2,783 "quote_count": 0,784 "incoming_link_count": 5,785 "reads": 17,786 "readers_count": 16,787 "score": 33.4,788 "yours": false,789 "topic_id": 135706,790 "topic_slug": "load-multiple-models-with-distributeddataparallel",791 "display_username": "Suho Cho",792 "primary_group_name": null,793 "flair_name": null,794 "flair_url": null,795 "flair_bg_color": null,796 "flair_color": null,797 "flair_group_id": null,798 "badges_granted": [],799 "version": 2,800 "can_edit": false,801 "can_delete": false,802 "can_recover": false,803 "can_see_hidden_post": false,804 "can_wiki": false,805 "read": true,806 "user_title": "",807 "reply_to_user": {808 "id": 17068,809 "username": "mrshenli",810 "name": "Shen Li",811 "avatar_template": "/user_avatar/discuss.pytorch.org/mrshenli/{size}/12220_2.png"812 },813 "bookmarked": false,814 "actions_summary": [],815 "moderator": false,816 "admin": false,817 "staff": false,818 "user_id": 7263,819 "hidden": false,820 "trust_level": 2,821 "deleted_at": null,822 "user_deleted": false,823 "edit_reason": null,824 "can_view_edit_history": true,825 "wiki": false,826 "post_url": "/t/load-multiple-models-with-distributeddataparallel/135706/3",827 "can_accept_answer": false,828 "can_unaccept_answer": false,829 "accepted_answer": true,830 "topic_accepted_answer": true831 },832 {833 "id": 315186,834 "name": "Shen Li",835 "username": "mrshenli",836 "avatar_template": "/user_avatar/discuss.pytorch.org/mrshenli/{size}/12220_2.png",837 "created_at": "2021-11-03T18:29:03.979Z",838 "cooked": "<blockquote>\n<p>I cannot understand why until now, but it is solved by using map_location while I load the model.</p>\n</blockquote>\n<p>I think this is because, all processes are trying to load the model to <code>cuda:0</code> by default if you don’t set <code>map_location</code> or <code>CUDA_VISIBLE_DEVICES</code>. BTW, does directly setting <code>map_location</code> to <code>device</code> work for you?</p>",839 "post_number": 4,840 "post_type": 1,841 "posts_count": 4,842 "updated_at": "2021-11-03T18:29:03.979Z",843 "reply_count": 0,844 "reply_to_post_number": 3,845 "quote_count": 0,846 "incoming_link_count": 2,847 "reads": 15,848 "readers_count": 14,849 "score": 13.0,850 "yours": false,851 "topic_id": 135706,852 "topic_slug": "load-multiple-models-with-distributeddataparallel",853 "display_username": "Shen Li",854 "primary_group_name": null,855 "flair_name": null,856 "flair_url": null,857 "flair_bg_color": null,858 "flair_color": null,859 "flair_group_id": null,860 "badges_granted": [],861 "version": 1,862 "can_edit": false,863 "can_delete": false,864 "can_recover": false,865 "can_see_hidden_post": false,866 "can_wiki": false,867 "read": true,868 "user_title": null,869 "reply_to_user": {870 "id": 7263,871 "username": "thecho7",872 "name": "Suho Cho",873 "avatar_template": "/letter_avatar_proxy/v4/letter/t/eada6e/{size}.png"874 },875 "bookmarked": false,876 "actions_summary": [],877 "moderator": false,878 "admin": false,879 "staff": false,880 "user_id": 17068,881 "hidden": false,882 "trust_level": 2,883 "deleted_at": null,884 "user_deleted": false,885 "edit_reason": null,886 "can_view_edit_history": true,887 "wiki": false,888 "post_url": "/t/load-multiple-models-with-distributeddataparallel/135706/4",889 "can_accept_answer": false,890 "can_unaccept_answer": false,891 "accepted_answer": false,892 "topic_accepted_answer": true893 }894 ],895 "stream": [896 314802,897 314851,898 314934,899 315186900 ]901 },902 "timeline_lookup": [903 [904 1,905 1454906 ],907 [908 2,909 1453910 ],911 [912 4,913 1452914 ]915 ],916 "suggested_topics": [917 {918 "fancy_title": "DistNetworkError when using multiprocessing_context parameter in pytorch dataloader",919 "id": 214577,920 "title": "DistNetworkError when using multiprocessing_context parameter in pytorch dataloader",921 "slug": "distnetworkerror-when-using-multiprocessing-context-parameter-in-pytorch-dataloader",922 "posts_count": 3,923 "reply_count": 1,924 "highest_post_number": 3,925 "image_url": null,926 "created_at": "2024-12-23T17:44:12.037Z",927 "last_posted_at": "2025-01-08T14:38:53.740Z",928 "bumped": true,929 "bumped_at": "2025-01-08T14:38:53.740Z",930 "archetype": "regular",931 "unseen": false,932 "pinned": false,933 "unpinned": null,934 "visible": true,935 "closed": false,936 "archived": false,937 "bookmarked": null,938 "liked": null,939 "tags_descriptions": {},940 "like_count": 0,941 "views": 243,942 "category_id": 12,943 "featured_link": null,944 "has_accepted_answer": true,945 "posters": [946 {947 "extras": "latest",948 "description": "Original Poster, Most Recent Poster, Accepted Answer",949 "user": {950 "id": 81406,951 "username": "forestbat",952 "name": "forestbat",953 "avatar_template": "/user_avatar/discuss.pytorch.org/forestbat/{size}/74440_2.png",954 "trust_level": 1955 }956 },957 {958 "extras": null,959 "description": "Frequent Poster",960 "user": {961 "id": 39542,962 "username": "H-Huang",963 "name": "Howard Huang",964 "avatar_template": "/user_avatar/discuss.pytorch.org/h-huang/{size}/35598_2.png",965 "trust_level": 2966 }967 }968 ]969 },970 {971 "fancy_title": "Issue with torchrun Multi-Node DDP Training: Process Group Not Destroyed Error",972 "id": 214810,973 "title": "Issue with torchrun Multi-Node DDP Training: Process Group Not Destroyed Error",974 "slug": "issue-with-torchrun-multi-node-ddp-training-process-group-not-destroyed-error",975 "posts_count": 1,976 "reply_count": 0,977 "highest_post_number": 1,978 "image_url": null,979 "created_at": "2024-12-31T05:21:39.398Z",980 "last_posted_at": "2024-12-31T05:21:39.448Z",981 "bumped": true,982 "bumped_at": "2024-12-31T05:22:54.024Z",983 "archetype": "regular",984 "unseen": false,985 "pinned": false,986 "unpinned": null,987 "visible": true,988 "closed": false,989 "archived": false,990 "bookmarked": null,991 "liked": null,992 "tags_descriptions": {},993 "like_count": 1,994 "views": 947,995 "category_id": 12,996 "featured_link": null,997 "has_accepted_answer": false,998 "posters": [999 {1000 "extras": "latest single",1001 "description": "Original Poster, Most Recent Poster",1002 "user": {1003 "id": 81801,1004 "username": "tnfks0606",1005 "name": "suran choi",1006 "avatar_template": "/user_avatar/discuss.pytorch.org/tnfks0606/{size}/74825_2.png",1007 "trust_level": 11008 }1009 }1010 ]1011 },1012 {1013 "fancy_title": "FP8 training with torchao but without torchtitan",1014 "id": 215152,1015 "title": "FP8 training with torchao but without torchtitan",1016 "slug": "fp8-training-with-torchao-but-without-torchtitan",1017 "posts_count": 3,1018 "reply_count": 1,1019 "highest_post_number": 4,1020 "image_url": null,1021 "created_at": "2025-01-09T05:37:00.335Z",1022 "last_posted_at": "2025-02-07T08:08:30.443Z",1023 "bumped": true,1024 "bumped_at": "2025-02-07T08:08:30.443Z",1025 "archetype": "regular",1026 "unseen": false,1027 "pinned": false,1028 "unpinned": null,1029 "visible": true,1030 "closed": false,1031 "archived": false,1032 "bookmarked": null,1033 "liked": null,1034 "tags_descriptions": {},1035 "like_count": 0,1036 "views": 386,1037 "category_id": 12,1038 "featured_link": null,1039 "has_accepted_answer": false,1040 "posters": [1041 {1042 "extras": "latest",1043 "description": "Original Poster, Most Recent Poster",1044 "user": {1045 "id": 57464,1046 "username": "cokespace2",1047 "name": "Vince Mo",1048 "avatar_template": "/user_avatar/discuss.pytorch.org/cokespace2/{size}/51244_2.png",1049 "trust_level": 21050 }1051 },1052 {1053 "extras": null,1054 "description": "Frequent Poster",1055 "user": {1056 "id": 24320,1057 "username": "supriyar",1058 "name": "",1059 "avatar_template": "/letter_avatar_proxy/v4/letter/s/db5fbb/{size}.png",1060 "trust_level": 21061 }1062 }1063 ]1064 },1065 {1066 "fancy_title": "Efficiently Training Multiple Large Models with PyTorch FSDP: Best Practices?",1067 "id": 215658,1068 "title": "Efficiently Training Multiple Large Models with PyTorch FSDP: Best Practices?",1069 "slug": "efficiently-training-multiple-large-models-with-pytorch-fsdp-best-practices",1070 "posts_count": 1,1071 "reply_count": 0,1072 "highest_post_number": 1,1073 "image_url": null,1074 "created_at": "2025-01-21T08:00:14.834Z",1075 "last_posted_at": "2025-01-21T08:00:14.877Z",1076 "bumped": true,1077 "bumped_at": "2025-01-21T08:00:14.877Z",1078 "archetype": "regular",1079 "unseen": false,1080 "pinned": false,1081 "unpinned": null,1082 "visible": true,1083 "closed": false,1084 "archived": false,1085 "bookmarked": null,1086 "liked": null,1087 "tags_descriptions": {},1088 "like_count": 0,1089 "views": 75,1090 "category_id": 12,1091 "featured_link": null,1092 "has_accepted_answer": false,1093 "posters": [1094 {1095 "extras": "latest single",1096 "description": "Original Poster, Most Recent Poster",1097 "user": {1098 "id": 10331,1099 "username": "erfan_mhi",1100 "name": "Erfan Miahi",1101 "avatar_template": "/user_avatar/discuss.pytorch.org/erfan_mhi/{size}/74886_2.png",1102 "trust_level": 11103 }1104 }1105 ]1106 },1107 {1108 "fancy_title": "NCCL Timeout only on H100s, not other hardware",1109 "id": 216332,1110 "title": "NCCL Timeout only on H100s, not other hardware",1111 "slug": "nccl-timeout-only-on-h100s-not-other-hardware",1112 "posts_count": 4,1113 "reply_count": 2,1114 "highest_post_number": 4,1115 "image_url": null,1116 "created_at": "2025-02-06T20:39:32.191Z",1117 "last_posted_at": "2025-02-06T21:34:25.937Z",1118 "bumped": true,1119 "bumped_at": "2025-02-06T21:34:25.937Z",1120 "archetype": "regular",1121 "unseen": false,1122 "pinned": false,1123 "unpinned": null,1124 "visible": true,1125 "closed": false,1126 "archived": false,1127 "bookmarked": null,1128 "liked": null,1129 "tags_descriptions": {},1130 "like_count": 1,1131 "views": 564,1132 "category_id": 12,1133 "featured_link": null,1134 "has_accepted_answer": false,1135 "posters": [1136 {1137 "extras": "latest",1138 "description": "Original Poster, Most Recent Poster",1139 "user": {1140 "id": 82492,1141 "username": "ekomp",1142 "name": "",1143 "avatar_template": "/user_avatar/discuss.pytorch.org/ekomp/{size}/75513_2.png",1144 "trust_level": 11145 }1146 },1147 {1148 "extras": null,1149 "description": "Frequent Poster",1150 "user": {1151 "id": 3534,1152 "username": "ptrblck",1153 "name": "",1154 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1155 "admin": true,1156 "moderator": true,1157 "trust_level": 21158 }1159 }1160 ]1161 }1162 ],1163 "tags_descriptions": {},1164 "fancy_title": "Load multiple models with DistributedDataParallel",1165 "id": 135706,1166 "title": "Load multiple models with DistributedDataParallel",1167 "posts_count": 4,1168 "created_at": "2021-11-02T07:55:56.274Z",1169 "views": 1304,1170 "reply_count": 2,1171 "like_count": 0,1172 "last_posted_at": "2021-11-03T18:29:03.979Z",1173 "visible": true,1174 "closed": false,1175 "archived": false,1176 "has_summary": false,1177 "archetype": "regular",1178 "slug": "load-multiple-models-with-distributeddataparallel",1179 "category_id": 12,1180 "word_count": 299,1181 "deleted_at": null,1182 "user_id": 7263,1183 "featured_link": null,1184 "pinned_globally": false,1185 "pinned_at": null,1186 "pinned_until": null,1187 "image_url": null,1188 "slow_mode_seconds": 0,1189 "draft": null,1190 "draft_key": "topic_135706",1191 "draft_sequence": null,1192 "unpinned": null,1193 "pinned": false,1194 "current_post_number": 1,1195 "highest_post_number": 4,1196 "deleted_by": null,1197 "actions_summary": [1198 {1199 "id": 4,1200 "count": 0,