Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 365237,7 "name": "Bryan",8 "username": "bryan123",9 "avatar_template": "/user_avatar/discuss.pytorch.org/bryan123/{size}/51979_2.png",10 "created_at": "2022-09-07T00:25:05.236Z",11 "cooked": "<p>How can I add dropout layers after every convolution layer in DenseNet201 pretrained if I want to keep the value of its parameters (weights/Biases)? (FYI, I wanted to add dropout layers between the convolutional layers in order to quantify MC-Dropout uncertainty during prediction).</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 4,15 "updated_at": "2022-09-07T00:25:05.236Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 430,20 "reads": 11,21 "readers_count": 10,22 "score": 2152.2,23 "yours": false,24 "topic_id": 160915,25 "topic_slug": "adding-dropout-in-densenet201-pretrained",26 "display_username": "Bryan",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 58183,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/adding-dropout-in-densenet201-pretrained/160915/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": true,60 "can_vote": false61 },62 {63 "id": 365239,64 "name": "",65 "username": "ptrblck",66 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",67 "created_at": "2022-09-07T01:08:13.718Z",68 "cooked": "<p>You could try to replace each original conv layer with a new <code>nn.Sequential</code> layer containing the new dropout layer as well as the pre-trained conv layer.<br>\nSomething like this should work:</p>\n<pre><code class=\"lang-python\">model = models.densenet201()\n\n# replace conv layer\nold_conv = model.features[0]\n\nmodel.features[0] = nn.Sequential(\n nn.Dropout(),\n old_conv)\n</code></pre>\n<p>Note that you are changing the model architecture and thus won’t be able to load a pre-trained <code>state_dict</code> to this manipulated model anymore so make sure to load the <code>state_dict</code> before adding the dropout layers or store the new <code>state_dict</code> afterwards.</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 4,72 "updated_at": "2022-09-07T01:08:13.718Z",73 "reply_count": 0,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 11,77 "reads": 11,78 "readers_count": 10,79 "score": 57.2,80 "yours": false,81 "topic_id": 160915,82 "topic_slug": "adding-dropout-in-densenet201-pretrained",83 "display_username": "",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 3534,105 "hidden": false,106 "trust_level": 2,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/adding-dropout-in-densenet201-pretrained/160915/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": true117 },118 {119 "id": 365241,120 "name": "Bryan",121 "username": "bryan123",122 "avatar_template": "/user_avatar/discuss.pytorch.org/bryan123/{size}/51979_2.png",123 "created_at": "2022-09-07T01:20:02.891Z",124 "cooked": "<p>Shouldn’t dropout be added to the dense layer as well? Would it be better to put dropout before or after Conv2d?</p>\n<p>Could you also show me how to load the state_dict before adding dropout layers?</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 4,128 "updated_at": "2022-09-07T01:30:22.317Z",129 "reply_count": 1,130 "reply_to_post_number": null,131 "quote_count": 0,132 "incoming_link_count": 6,133 "reads": 11,134 "readers_count": 10,135 "score": 37.2,136 "yours": false,137 "topic_id": 160915,138 "topic_slug": "adding-dropout-in-densenet201-pretrained",139 "display_username": "Bryan",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 2,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": null,155 "bookmarked": false,156 "actions_summary": [],157 "moderator": false,158 "admin": false,159 "staff": false,160 "user_id": 58183,161 "hidden": false,162 "trust_level": 1,163 "deleted_at": null,164 "user_deleted": false,165 "edit_reason": null,166 "can_view_edit_history": true,167 "wiki": false,168 "post_url": "/t/adding-dropout-in-densenet201-pretrained/160915/3",169 "can_accept_answer": false,170 "can_unaccept_answer": false,171 "accepted_answer": false,172 "topic_accepted_answer": true173 },174 {175 "id": 365250,176 "name": "",177 "username": "ptrblck",178 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",179 "created_at": "2022-09-07T03:17:55.959Z",180 "cooked": "<p>You can add the dropout layers wherever you think it would work and my code snippet is just one example how to add it before one conv layer.</p>\n<p>If you want to use the pretrained model from <code>torchvision</code> use:</p>\n<pre><code class=\"lang-python\">model = models.densenet201(pretrained=True)\n</code></pre>\n<p>otherwise if you want to load a <code>state_dict</code> that you’ve stored after training the model load it via:</p>\n<pre><code class=\"lang-python\">model = models.densenet201()\nmodel.load_state_dict(torch.load(path_to_state_dict))\n</code></pre>\n<p>before changing the architecture.</p>",181 "post_number": 4,182 "post_type": 1,183 "posts_count": 4,184 "updated_at": "2022-09-07T03:17:55.959Z",185 "reply_count": 0,186 "reply_to_post_number": 3,187 "quote_count": 0,188 "incoming_link_count": 14,189 "reads": 8,190 "readers_count": 7,191 "score": 71.6,192 "yours": false,193 "topic_id": 160915,194 "topic_slug": "adding-dropout-in-densenet201-pretrained",195 "display_username": "",196 "primary_group_name": null,197 "flair_name": null,198 "flair_url": null,199 "flair_bg_color": null,200 "flair_color": null,201 "flair_group_id": null,202 "badges_granted": [],203 "version": 1,204 "can_edit": false,205 "can_delete": false,206 "can_recover": false,207 "can_see_hidden_post": false,208 "can_wiki": false,209 "read": true,210 "user_title": "",211 "reply_to_user": {212 "id": 58183,213 "username": "bryan123",214 "name": "Bryan",215 "avatar_template": "/user_avatar/discuss.pytorch.org/bryan123/{size}/51979_2.png"216 },217 "bookmarked": false,218 "actions_summary": [],219 "moderator": true,220 "admin": true,221 "staff": true,222 "user_id": 3534,223 "hidden": false,224 "trust_level": 2,225 "deleted_at": null,226 "user_deleted": false,227 "edit_reason": null,228 "can_view_edit_history": true,229 "wiki": false,230 "post_url": "/t/adding-dropout-in-densenet201-pretrained/160915/4",231 "can_accept_answer": false,232 "can_unaccept_answer": false,233 "accepted_answer": true,234 "topic_accepted_answer": true235 }236 ],237 "stream": [238 365237,239 365239,240 365241,241 365250242 ]243 },244 "timeline_lookup": [245 [246 1,247 1145248 ]249 ],250 "suggested_topics": [251 {252 "fancy_title": "Torch.nn.functional.grid_sample with align_corners=False",253 "id": 219209,254 "title": "Torch.nn.functional.grid_sample with align_corners=False",255 "slug": "torch-nn-functional-grid-sample-with-align-corners-false",256 "posts_count": 1,257 "reply_count": 0,258 "highest_post_number": 1,259 "image_url": null,260 "created_at": "2025-04-17T14:38:04.220Z",261 "last_posted_at": "2025-04-17T14:38:04.269Z",262 "bumped": true,263 "bumped_at": "2025-04-17T14:38:04.269Z",264 "archetype": "regular",265 "unseen": false,266 "pinned": false,267 "unpinned": null,268 "visible": true,269 "closed": false,270 "archived": false,271 "bookmarked": null,272 "liked": null,273 "tags_descriptions": {},274 "like_count": 0,275 "views": 44,276 "category_id": 1,277 "featured_link": null,278 "has_accepted_answer": false,279 "posters": [280 {281 "extras": "latest single",282 "description": "Original Poster, Most Recent Poster",283 "user": {284 "id": 83665,285 "username": "Pepper9283",286 "name": null,287 "avatar_template": "/letter_avatar_proxy/v4/letter/p/848f3c/{size}.png",288 "trust_level": 1289 }290 }291 ]292 },293 {294 "fancy_title": "Torch.cuda.is_available() returns False with cuda12.3",295 "id": 215442,296 "title": "Torch.cuda.is_available() returns False with cuda12.3",297 "slug": "torch-cuda-is-available-returns-false-with-cuda12-3",298 "posts_count": 1,299 "reply_count": 0,300 "highest_post_number": 1,301 "image_url": null,302 "created_at": "2025-01-16T00:24:21.813Z",303 "last_posted_at": "2025-01-16T00:24:21.857Z",304 "bumped": true,305 "bumped_at": "2025-01-16T00:24:21.857Z",306 "archetype": "regular",307 "unseen": false,308 "pinned": false,309 "unpinned": null,310 "visible": true,311 "closed": false,312 "archived": false,313 "bookmarked": null,314 "liked": null,315 "tags_descriptions": {},316 "like_count": 0,317 "views": 101,318 "category_id": 1,319 "featured_link": null,320 "has_accepted_answer": false,321 "posters": [322 {323 "extras": "latest single",324 "description": "Original Poster, Most Recent Poster",325 "user": {326 "id": 82111,327 "username": "leewww",328 "name": "",329 "avatar_template": "/letter_avatar_proxy/v4/letter/l/76d3ee/{size}.png",330 "trust_level": 1331 }332 }333 ]334 },335 {336 "fancy_title": "Multithreading in dataloader workers",337 "id": 215391,338 "title": "Multithreading in dataloader workers",339 "slug": "multithreading-in-dataloader-workers",340 "posts_count": 1,341 "reply_count": 0,342 "highest_post_number": 1,343 "image_url": null,344 "created_at": "2025-01-14T19:37:36.279Z",345 "last_posted_at": "2025-01-14T19:37:36.321Z",346 "bumped": true,347 "bumped_at": "2025-01-14T19:51:42.020Z",348 "archetype": "regular",349 "unseen": false,350 "pinned": false,351 "unpinned": null,352 "visible": true,353 "closed": false,354 "archived": false,355 "bookmarked": null,356 "liked": null,357 "tags_descriptions": {},358 "like_count": 0,359 "views": 123,360 "category_id": 1,361 "featured_link": null,362 "has_accepted_answer": false,363 "posters": [364 {365 "extras": "latest single",366 "description": "Original Poster, Most Recent Poster",367 "user": {368 "id": 71847,369 "username": "Tristan_Brugere",370 "name": "Tristan Brugere",371 "avatar_template": "/user_avatar/discuss.pytorch.org/tristan_brugere/{size}/66365_2.png",372 "trust_level": 1373 }374 }375 ]376 },377 {378 "fancy_title": "DDP leads to Out of Memory error",379 "id": 213422,380 "title": "DDP leads to Out of Memory error",381 "slug": "ddp-leads-to-out-of-memory-error",382 "posts_count": 1,383 "reply_count": 0,384 "highest_post_number": 1,385 "image_url": null,386 "created_at": "2024-11-25T19:51:07.570Z",387 "last_posted_at": "2024-11-25T19:51:07.613Z",388 "bumped": true,389 "bumped_at": "2024-11-25T19:51:07.613Z",390 "archetype": "regular",391 "unseen": false,392 "pinned": false,393 "unpinned": null,394 "visible": true,395 "closed": false,396 "archived": false,397 "bookmarked": null,398 "liked": null,399 "tags_descriptions": {},400 "like_count": 0,401 "views": 58,402 "category_id": 1,403 "featured_link": null,404 "has_accepted_answer": false,405 "posters": [406 {407 "extras": "latest single",408 "description": "Original Poster, Most Recent Poster",409 "user": {410 "id": 37254,411 "username": "enterthevoidf22",412 "name": "",413 "avatar_template": "/user_avatar/discuss.pytorch.org/enterthevoidf22/{size}/29338_2.png",414 "trust_level": 2415 }416 }417 ]418 },419 {420 "fancy_title": "Initialise the parameters of a NN and then train it",421 "id": 221334,422 "title": "Initialise the parameters of a NN and then train it",423 "slug": "initialise-the-parameters-of-a-nn-and-then-train-it",424 "posts_count": 6,425 "reply_count": 2,426 "highest_post_number": 6,427 "image_url": null,428 "created_at": "2025-07-07T13:23:13.158Z",429 "last_posted_at": "2025-07-09T12:34:39.704Z",430 "bumped": true,431 "bumped_at": "2025-07-09T12:34:39.704Z",432 "archetype": "regular",433 "unseen": false,434 "pinned": false,435 "unpinned": null,436 "visible": true,437 "closed": false,438 "archived": false,439 "bookmarked": null,440 "liked": null,441 "tags_descriptions": {},442 "like_count": 1,443 "views": 47,444 "category_id": 1,445 "featured_link": null,446 "has_accepted_answer": false,447 "posters": [448 {449 "extras": null,450 "description": "Original Poster",451 "user": {452 "id": 84975,453 "username": "fx12345",454 "name": "",455 "avatar_template": "/letter_avatar_proxy/v4/letter/f/58f4c7/{size}.png",456 "trust_level": 1457 }458 },459 {460 "extras": null,461 "description": "Frequent Poster",462 "user": {463 "id": 84935,464 "username": "paulk",465 "name": "",466 "avatar_template": "/letter_avatar_proxy/v4/letter/p/67e7ee/{size}.png",467 "trust_level": 2468 }469 },470 {471 "extras": "latest",472 "description": "Most Recent Poster",473 "user": {474 "id": 84484,475 "username": "Dhia-naouali",476 "name": "Dhia naouali",477 "avatar_template": "/user_avatar/discuss.pytorch.org/dhia-naouali/{size}/77193_2.png",478 "trust_level": 2479 }480 }481 ]482 }483 ],484 "tags_descriptions": {},485 "fancy_title": "Adding Dropout in DenseNet201 Pretrained",486 "id": 160915,487 "title": "Adding Dropout in DenseNet201 Pretrained",488 "posts_count": 4,489 "created_at": "2022-09-07T00:25:05.187Z",490 "views": 1032,491 "reply_count": 1,492 "like_count": 0,493 "last_posted_at": "2022-09-07T03:17:55.959Z",494 "visible": true,495 "closed": false,496 "archived": false,497 "has_summary": false,498 "archetype": "regular",499 "slug": "adding-dropout-in-densenet201-pretrained",500 "category_id": 1,501 "word_count": 253,502 "deleted_at": null,503 "user_id": 58183,504 "featured_link": null,505 "pinned_globally": false,506 "pinned_at": null,507 "pinned_until": null,508 "image_url": null,509 "slow_mode_seconds": 0,510 "draft": null,511 "draft_key": "topic_160915",512 "draft_sequence": null,513 "unpinned": null,514 "pinned": false,515 "current_post_number": 1,516 "highest_post_number": 4,517 "deleted_by": null,518 "actions_summary": [519 {520 "id": 4,521 "count": 0,522 "hidden": false,523 "can_act": false524 },525 {526 "id": 8,527 "count": 0,528 "hidden": false,529 "can_act": false530 },531 {532 "id": 10,533 "count": 0,534 "hidden": false,535 "can_act": false536 },537 {538 "id": 7,539 "count": 0,540 "hidden": false,541 "can_act": false542 }543 ],544 "chunk_size": 20,545 "bookmarked": false,546 "topic_timer": null,547 "message_bus_last_id": 0,548 "participant_count": 2,549 "show_read_indicator": false,550 "thumbnails": null,551 "slow_mode_enabled_until": null,552 "accepted_answer": {553 "post_number": 4,554 "username": "ptrblck",555 "name": "",556 "excerpt": "You can add the dropout layers wherever you think it would work and my code snippet is just one example how to add it before one conv layer. \nIf you want to use the pretrained model from torchvision use: \nmodel = models.densenet201(pretrained=True)\n\notherwise if you want to load a state_dict that yo…"557 },558 "can_vote": false,559 "vote_count": 0,560 "user_voted": false,561 "discourse_zendesk_plugin_zendesk_id": null,562 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",563 "details": {564 "can_edit": false,565 "notification_level": 1,566 "participants": [567 {568 "id": 3534,569 "username": "ptrblck",570 "name": "",571 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",572 "post_count": 2,573 "primary_group_name": null,574 "flair_name": null,575 "flair_url": null,576 "flair_color": null,577 "flair_bg_color": null,578 "flair_group_id": null,579 "admin": true,580 "moderator": true,581 "trust_level": 2582 },583 {584 "id": 58183,585 "username": "bryan123",586 "name": "Bryan",587 "avatar_template": "/user_avatar/discuss.pytorch.org/bryan123/{size}/51979_2.png",588 "post_count": 2,589 "primary_group_name": null,590 "flair_name": null,591 "flair_url": null,592 "flair_color": null,593 "flair_bg_color": null,594 "flair_group_id": null,595 "trust_level": 1596 }597 ],598 "created_by": {599 "id": 58183,600 "username": "bryan123",601 "name": "Bryan",602 "avatar_template": "/user_avatar/discuss.pytorch.org/bryan123/{size}/51979_2.png"603 },604 "last_poster": {605 "id": 3534,606 "username": "ptrblck",607 "name": "",608 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"609 }610 },611 "bookmarks": []612 },613 {614 "post_stream": {615 "posts": [616 {617 "id": 364813,618 "name": "",619 "username": "researcher2",620 "avatar_template": "/letter_avatar_proxy/v4/letter/r/ecae2f/{size}.png",621 "created_at": "2022-09-04T07:53:40.117Z",622 "cooked": "<p>I have been using FSDP on 1.11 stable, checkpointing on each device with the wrapped model: model.state_dict(). When loading the checkpoint I would initialize the model, wrap with FSDP then do load_state_dict on the wrapped model on each device, and training would resume successfully.</p>\n<p>I’m looking to do some evals and can probably fit the entire model on a single device, and therefore need to unshard things.</p>\n<p>Are these checkpoints usable with any of the apis available in 1.12? load_state_dict, load_local_state_dict, or load_shared_state_dict?</p>\n<p>Thank You</p>",623 "post_number": 1,624 "post_type": 1,625 "posts_count": 4,626 "updated_at": "2022-09-04T07:54:30.072Z",627 "reply_count": 0,628 "reply_to_post_number": null,629 "quote_count": 0,630 "incoming_link_count": 183,631 "reads": 13,632 "readers_count": 12,633 "score": 917.6,634 "yours": false,635 "topic_id": 160697,636 "topic_slug": "migrating-from-1-11-to-1-12-fsdp",637 "display_username": "",638 "primary_group_name": null,639 "flair_name": null,640 "flair_url": null,641 "flair_bg_color": null,642 "flair_color": null,643 "flair_group_id": null,644 "badges_granted": [],645 "version": 1,646 "can_edit": false,647 "can_delete": false,648 "can_recover": false,649 "can_see_hidden_post": false,650 "can_wiki": false,651 "read": true,652 "user_title": null,653 "bookmarked": false,654 "actions_summary": [],655 "moderator": false,656 "admin": false,657 "staff": false,658 "user_id": 59077,659 "hidden": false,660 "trust_level": 0,661 "deleted_at": null,662 "user_deleted": false,663 "edit_reason": null,664 "can_view_edit_history": true,665 "wiki": false,666 "post_url": "/t/migrating-from-1-11-to-1-12-fsdp/160697/1",667 "can_accept_answer": false,668 "can_unaccept_answer": false,669 "accepted_answer": false,670 "topic_accepted_answer": null,671 "can_vote": false672 },673 {674 "id": 365151,675 "name": "Howard Huang",676 "username": "H-Huang",677 "avatar_template": "/user_avatar/discuss.pytorch.org/h-huang/{size}/35598_2.png",678 "created_at": "2022-09-06T15:13:46.575Z",679 "cooked": "<p>cc <a class=\"mention\" href=\"/u/yanli_zhao\">@Yanli_Zhao</a> <a class=\"mention\" href=\"/u/rvarm1\">@rvarm1</a> <a class=\"mention\" href=\"/u/agu\">@agu</a> for FSDP related question</p>",680 "post_number": 2,681 "post_type": 1,682 "posts_count": 4,683 "updated_at": "2022-09-06T15:13:46.575Z",684 "reply_count": 0,685 "reply_to_post_number": null,686 "quote_count": 0,687 "incoming_link_count": 4,688 "reads": 14,689 "readers_count": 13,690 "score": 22.8,691 "yours": false,692 "topic_id": 160697,693 "topic_slug": "migrating-from-1-11-to-1-12-fsdp",694 "display_username": "Howard Huang",695 "primary_group_name": null,696 "flair_name": null,697 "flair_url": null,698 "flair_bg_color": null,699 "flair_color": null,700 "flair_group_id": null,701 "badges_granted": [],702 "version": 1,703 "can_edit": false,704 "can_delete": false,705 "can_recover": false,706 "can_see_hidden_post": false,707 "can_wiki": false,708 "read": true,709 "user_title": null,710 "bookmarked": false,711 "actions_summary": [],712 "moderator": false,713 "admin": false,714 "staff": false,715 "user_id": 39542,716 "hidden": false,717 "trust_level": 2,718 "deleted_at": null,719 "user_deleted": false,720 "edit_reason": null,721 "can_view_edit_history": true,722 "wiki": false,723 "post_url": "/t/migrating-from-1-11-to-1-12-fsdp/160697/2",724 "can_accept_answer": false,725 "can_unaccept_answer": false,726 "accepted_answer": false,727 "topic_accepted_answer": null728 },729 {730 "id": 365213,731 "name": "Rodrigo Kumpera",732 "username": "kumpera",733 "avatar_template": "/user_avatar/discuss.pytorch.org/kumpera/{size}/48311_2.png",734 "created_at": "2022-09-06T21:16:19.657Z",735 "cooked": "<p>If you save your model that way you won’t be able to change the cluster topology later.</p>\n<p>There are essentially two recommended ways to checkpoint a FSDP model:</p>\n<p>Save the whole model from rank0:</p>\n<pre><code class=\"lang-auto\">import torch.distributed as dist\nfrom torch.distributed.fsdp import FullyShardedDataParallel as FSDP\n\nmodel = ...\n# Save\nwith FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT):\n state_dict = model.state_dict()\n if dist.get_rank() == 0:\n torch.save(state_dict, \"checkpoint.pt\")\n\n# Load\nStateDictType.FULL_STATE_DICT):\n model.load_state_dict(torch.load(\"checkpoint.pt\"))\n</code></pre>\n<p>Save in a distributed fashion using the experimental distributed checkpointing API:</p>\n<pre><code class=\"lang-auto\">import torch.distributed as dist\nfrom torch.distributed.fsdp import FullyShardedDataParallel as FSDP\nimport torch.distributed._shard.checkpoint as dist_cp\n\nmodel = ...\n# Save\nwith FSDP.state_dict_type(model, \nStateDictType.SHARDED_STATE_DICT):\n checkpoint = model.state_dict()\n dist_cp.save_state_dict(\n state_dict=checkpoint,\n storage_writer=dist_cp.FileSystemWriter(\"checkpoint\")\n )\n\n# Load\nwith FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT):\n checkpoint = model.state_dict()\n dist_cp.load_state_dict(\n state_dict=checkpoint,\n storage_reader=dist_cp.FileSystemReader(\"checkpoint\")\n )\n model.load_state_dict(checkpoint)\n</code></pre>\n<p>While you can use StateDictType.LOCAL_STATE_DICT and checkpoint in a similar way to how you’re doing today, it won’t handle the scenario of changing your cluster topology (IE, go from N ranks to single rank).</p>\n<p>If you have further questions, and have a slack account, feel free to ping me with any further questions on checkpointing - I’d love to hear about your experience with it.</p>",736 "post_number": 3,737 "post_type": 1,738 "posts_count": 4,739 "updated_at": "2022-09-06T21:16:19.657Z",740 "reply_count": 1,741 "reply_to_post_number": null,742 "quote_count": 0,743 "incoming_link_count": 18,744 "reads": 14,745 "readers_count": 13,746 "score": 112.8,747 "yours": false,748 "topic_id": 160697,749 "topic_slug": "migrating-from-1-11-to-1-12-fsdp",750 "display_username": "Rodrigo Kumpera",751 "primary_group_name": null,752 "flair_name": null,753 "flair_url": null,754 "flair_bg_color": null,755 "flair_color": null,756 "flair_group_id": null,757 "badges_granted": [],758 "version": 1,759 "can_edit": false,760 "can_delete": false,761 "can_recover": false,762 "can_see_hidden_post": false,763 "can_wiki": false,764 "read": true,765 "user_title": null,766 "bookmarked": false,767 "actions_summary": [768 {769 "id": 2,770 "count": 1771 }772 ],773 "moderator": false,774 "admin": false,775 "staff": false,776 "user_id": 54735,777 "hidden": false,778 "trust_level": 2,779 "deleted_at": null,780 "user_deleted": false,781 "edit_reason": null,782 "can_view_edit_history": true,783 "wiki": false,784 "post_url": "/t/migrating-from-1-11-to-1-12-fsdp/160697/3",785 "can_accept_answer": false,786 "can_unaccept_answer": false,787 "accepted_answer": false,788 "topic_accepted_answer": null789 },790 {791 "id": 365248,792 "name": "",793 "username": "researcher2",794 "avatar_template": "/letter_avatar_proxy/v4/letter/r/ecae2f/{size}.png",795 "created_at": "2022-09-07T02:59:03.263Z",796 "cooked": "<p>Ok thank you.</p>\n<p>I did find something in the 1.11 tests that might have recovered the params: <a href=\"https://github.com/pytorch/pytorch/blob/bc2c6edaf163b1a1330e37a6e34caf8c553e4755/torch/testing/_internal/common_fsdp.py#L38\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/common_fsdp.py at bc2c6edaf163b1a1330e37a6e34caf8c553e4755 · pytorch/pytorch · GitHub</a></p>\n<p>After running this function on a model you can call state_dict(), and remove the “_fsdp_wrapped_module._fpw_module.” from the keys to produce something that looks like an original model checkpoint, however my testing indicates the parameters are different so I’m not sure it can be relied upon. I will do a little more testing.</p>\n<p>For my smaller models I can retrain fairly cheaply and am using the FULL_STATE_DICT which works very nicely!</p>",797 "post_number": 4,798 "post_type": 1,799 "posts_count": 4,800 "updated_at": "2022-09-07T03:01:18.892Z",801 "reply_count": 0,802 "reply_to_post_number": 3,803 "quote_count": 0,804 "incoming_link_count": 3,805 "reads": 11,806 "readers_count": 10,807 "score": 17.2,808 "yours": false,809 "topic_id": 160697,810 "topic_slug": "migrating-from-1-11-to-1-12-fsdp",811 "display_username": "",812 "primary_group_name": null,813 "flair_name": null,814 "flair_url": null,815 "flair_bg_color": null,816 "flair_color": null,817 "flair_group_id": null,818 "badges_granted": [],819 "version": 2,820 "can_edit": false,821 "can_delete": false,822 "can_recover": false,823 "can_see_hidden_post": false,824 "can_wiki": false,825 "link_counts": [826 {827 "url": "https://github.com/pytorch/pytorch/blob/bc2c6edaf163b1a1330e37a6e34caf8c553e4755/torch/testing/_internal/common_fsdp.py#L38",828 "internal": false,829 "reflection": false,830 "title": "pytorch/common_fsdp.py at bc2c6edaf163b1a1330e37a6e34caf8c553e4755 · pytorch/pytorch · GitHub",831 "clicks": 4832 }833 ],834 "read": true,835 "user_title": null,836 "reply_to_user": {837 "id": 54735,838 "username": "kumpera",839 "name": "Rodrigo Kumpera",840 "avatar_template": "/user_avatar/discuss.pytorch.org/kumpera/{size}/48311_2.png"841 },842 "bookmarked": false,843 "actions_summary": [],844 "moderator": false,845 "admin": false,846 "staff": false,847 "user_id": 59077,848 "hidden": false,849 "trust_level": 0,850 "deleted_at": null,851 "user_deleted": false,852 "edit_reason": null,853 "can_view_edit_history": true,854 "wiki": false,855 "post_url": "/t/migrating-from-1-11-to-1-12-fsdp/160697/4",856 "can_accept_answer": false,857 "can_unaccept_answer": false,858 "accepted_answer": false,859 "topic_accepted_answer": null860 }861 ],862 "stream": [863 364813,864 365151,865 365213,866 365248867 ]868 },869 "timeline_lookup": [870 [871 1,872 1148873 ],874 [875 2,876 1145877 ]878 ],879 "suggested_topics": [880 {881 "fancy_title": "Segfault during torch.save",882 "id": 216795,883 "title": "Segfault during torch.save",884 "slug": "segfault-during-torch-save",885 "posts_count": 3,886 "reply_count": 1,887 "highest_post_number": 3,888 "image_url": null,889 "created_at": "2025-02-18T00:44:16.977Z",890 "last_posted_at": "2025-02-21T02:37:43.749Z",891 "bumped": true,892 "bumped_at": "2025-02-21T02:37:43.749Z",893 "archetype": "regular",894 "unseen": false,895 "pinned": false,896 "unpinned": null,897 "visible": true,898 "closed": false,899 "archived": false,900 "bookmarked": null,901 "liked": null,902 "tags_descriptions": {},903 "like_count": 0,904 "views": 147,905 "category_id": 12,906 "featured_link": null,907 "has_accepted_answer": false,908 "posters": [909 {910 "extras": null,911 "description": "Original Poster",912 "user": {913 "id": 82747,914 "username": "ryxli",915 "name": "",916 "avatar_template": "/user_avatar/discuss.pytorch.org/ryxli/{size}/75709_2.png",917 "trust_level": 1918 }919 },920 {921 "extras": "latest",922 "description": "Most Recent Poster",923 "user": {924 "id": 3534,925 "username": "ptrblck",926 "name": "",927 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",928 "admin": true,929 "moderator": true,930 "trust_level": 2931 }932 }933 ]934 },935 {936 "fancy_title": "Subprocess groups w/ DeviceMesh Blocking",937 "id": 219412,938 "title": "Subprocess groups w/ DeviceMesh Blocking",939 "slug": "subprocess-groups-w-devicemesh-blocking",940 "posts_count": 3,941 "reply_count": 0,942 "highest_post_number": 3,943 "image_url": null,944 "created_at": "2025-04-24T06:25:20.807Z",945 "last_posted_at": "2025-05-03T06:57:06.953Z",946 "bumped": true,947 "bumped_at": "2025-05-03T06:57:06.953Z",948 "archetype": "regular",949 "unseen": false,950 "pinned": false,951 "unpinned": null,952 "visible": true,953 "closed": false,954 "archived": false,955 "bookmarked": null,956 "liked": null,957 "tags_descriptions": {},958 "like_count": 0,959 "views": 137,960 "category_id": 12,961 "featured_link": null,962 "has_accepted_answer": false,963 "posters": [964 {965 "extras": null,966 "description": "Original Poster",967 "user": {968 "id": 83737,969 "username": "kr12",970 "name": "",971 "avatar_template": "/user_avatar/discuss.pytorch.org/kr12/{size}/76557_2.png",972 "trust_level": 1973 }974 },975 {976 "extras": null,977 "description": "Frequent Poster",978 "user": {979 "id": 54320,980 "username": "fduwjj",981 "name": "Hugo",982 "avatar_template": "/user_avatar/discuss.pytorch.org/fduwjj/{size}/47855_2.png",983 "trust_level": 2984 }985 },986 {987 "extras": "latest",988 "description": "Most Recent Poster",989 "user": {990 "id": 84129,991 "username": "dest1n1s",992 "name": "Dest1n1",993 "avatar_template": "/letter_avatar_proxy/v4/letter/d/3e96dc/{size}.png",994 "trust_level": 1995 }996 }997 ]998 },999 {1000 "fancy_title": "DDP device hanging before running torch.dist.all_reduce()",1001 "id": 213414,1002 "title": "DDP device hanging before running torch.dist.all_reduce()",1003 "slug": "ddp-device-hanging-before-running-torch-dist-all-reduce",1004 "posts_count": 1,1005 "reply_count": 0,1006 "highest_post_number": 1,1007 "image_url": null,1008 "created_at": "2024-11-25T17:34:16.233Z",1009 "last_posted_at": "2024-11-25T17:34:16.358Z",1010 "bumped": true,1011 "bumped_at": "2024-11-25T17:49:04.713Z",1012 "archetype": "regular",1013 "unseen": false,1014 "pinned": false,1015 "unpinned": null,1016 "visible": true,1017 "closed": false,1018 "archived": false,1019 "bookmarked": null,1020 "liked": null,1021 "tags_descriptions": {},1022 "like_count": 0,1023 "views": 152,1024 "category_id": 12,1025 "featured_link": null,1026 "has_accepted_answer": false,1027 "posters": [1028 {1029 "extras": "latest single",1030 "description": "Original Poster, Most Recent Poster",1031 "user": {1032 "id": 80741,1033 "username": "jacksettles",1034 "name": "Jack",1035 "avatar_template": "/letter_avatar_proxy/v4/letter/j/ac91a4/{size}.png",1036 "trust_level": 01037 }1038 }1039 ]1040 },1041 {1042 "fancy_title": "FSDP issue with invertible networks",1043 "id": 215228,1044 "title": "FSDP issue with invertible networks",1045 "slug": "fsdp-issue-with-invertible-networks",1046 "posts_count": 2,1047 "reply_count": 0,1048 "highest_post_number": 4,1049 "image_url": null,1050 "created_at": "2025-01-10T18:29:10.087Z",1051 "last_posted_at": "2025-01-17T21:23:20.825Z",1052 "bumped": true,1053 "bumped_at": "2025-01-17T21:23:20.825Z",1054 "archetype": "regular",1055 "unseen": false,1056 "pinned": false,1057 "unpinned": null,1058 "visible": true,1059 "closed": false,1060 "archived": false,1061 "bookmarked": null,1062 "liked": null,1063 "tags_descriptions": {},1064 "like_count": 1,1065 "views": 237,1066 "category_id": 12,1067 "featured_link": null,1068 "has_accepted_answer": true,1069 "posters": [1070 {1071 "extras": "latest single",1072 "description": "Original Poster, Most Recent Poster, Accepted Answer",1073 "user": {1074 "id": 81997,1075 "username": "gseastream",1076 "name": "Grant Seastream",1077 "avatar_template": "/user_avatar/discuss.pytorch.org/gseastream/{size}/75026_2.png",1078 "trust_level": 11079 }1080 }1081 ]1082 },1083 {1084 "fancy_title": "How to handle training of few layers with DDP",1085 "id": 219979,1086 "title": "How to handle training of few layers with DDP",1087 "slug": "how-to-handle-training-of-few-layers-with-ddp",1088 "posts_count": 3,1089 "reply_count": 1,1090 "highest_post_number": 3,1091 "image_url": null,1092 "created_at": "2025-05-13T08:17:42.196Z",1093 "last_posted_at": "2025-05-14T08:40:41.263Z",1094 "bumped": true,1095 "bumped_at": "2025-05-14T08:40:41.263Z",1096 "archetype": "regular",1097 "unseen": false,1098 "pinned": false,1099 "unpinned": null,1100 "visible": true,1101 "closed": false,1102 "archived": false,1103 "bookmarked": null,1104 "liked": null,1105 "tags_descriptions": {},1106 "like_count": 0,1107 "views": 62,1108 "category_id": 12,1109 "featured_link": null,1110 "has_accepted_answer": true,1111 "posters": [1112 {1113 "extras": "latest",1114 "description": "Original Poster, Most Recent Poster",1115 "user": {1116 "id": 21407,1117 "username": "Guillaume_Jeanneret",1118 "name": "Guillaume Jeanneret",1119 "avatar_template": "/user_avatar/discuss.pytorch.org/guillaume_jeanneret/{size}/14837_2.png",1120 "trust_level": 11121 }1122 },1123 {1124 "extras": null,1125 "description": "Frequent Poster, Accepted Answer",1126 "user": {1127 "id": 3534,1128 "username": "ptrblck",1129 "name": "",1130 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1131 "admin": true,1132 "moderator": true,1133 "trust_level": 21134 }1135 }1136 ]1137 }1138 ],1139 "tags_descriptions": {},1140 "fancy_title": "Migrating From 1.11 to 1.12 FSDP",1141 "id": 160697,1142 "title": "Migrating From 1.11 to 1.12 FSDP",1143 "posts_count": 4,1144 "created_at": "2022-09-04T07:53:40.064Z",1145 "views": 720,1146 "reply_count": 1,1147 "like_count": 1,1148 "last_posted_at": "2022-09-07T02:59:03.263Z",1149 "visible": true,1150 "closed": false,1151 "archived": false,1152 "has_summary": false,1153 "archetype": "regular",1154 "slug": "migrating-from-1-11-to-1-12-fsdp",1155 "category_id": 12,1156 "word_count": 416,1157 "deleted_at": null,1158 "user_id": 59077,1159 "featured_link": null,1160 "pinned_globally": false,1161 "pinned_at": null,1162 "pinned_until": null,1163 "image_url": null,1164 "slow_mode_seconds": 0,1165 "draft": null,1166 "draft_key": "topic_160697",1167 "draft_sequence": null,1168 "unpinned": null,1169 "pinned": false,1170 "current_post_number": 1,1171 "highest_post_number": 4,1172 "deleted_by": null,1173 "actions_summary": [1174 {1175 "id": 4,1176 "count": 0,1177 "hidden": false,1178 "can_act": false1179 },1180 {1181 "id": 8,1182 "count": 0,1183 "hidden": false,1184 "can_act": false1185 },1186 {1187 "id": 10,1188 "count": 0,1189 "hidden": false,1190 "can_act": false1191 },1192 {1193 "id": 7,1194 "count": 0,1195 "hidden": false,1196 "can_act": false1197 }1198 ],1199 "chunk_size": 20,1200 "bookmarked": false,