Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 466160,7 "name": "Chris YU",8 "username": "Chris_YU",9 "avatar_template": "/user_avatar/discuss.pytorch.org/chris_yu/{size}/75844_2.png",10 "created_at": "2025-02-24T07:26:18.787Z",11 "cooked": "<p>I know this is a pretty old topic. I am using model parallel with DDP: the model is partitioned into four parts, each residing on a GPU. When I restart training from a checkpoint, it gives Cuda out of memory error on GPU1 (largest model partition). I am using AdamW as optimizer and gradient checkpointing. The loading process looks like below:</p>\n<pre><code class=\"lang-auto\">checkpoint = torch.load('checkpoint.pth', map_location='cpu')\nmodel = MyModel()\nmodel.load_state_dict(checkpoint['model'])\noptimizer.load_state_dict(checkpoint['optimizer'])\n\nmodel.to_device(device_list) #distribute model to 4 GPU\nmodel = DDP(model, device_ids=None, output_device=None)\n\ndel checkpoint\ntorch.cuda.empty_cache()\n</code></pre>\n<p>Tried different methods online but nothing worked.</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 1,15 "updated_at": "2025-02-24T07:28:14.139Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 11,20 "reads": 6,21 "readers_count": 5,22 "score": 56.2,23 "yours": false,24 "topic_id": 217083,25 "topic_slug": "cuda-out-of-memory-when-restart-from-checkpoint",26 "display_username": "Chris YU",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 82890,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/cuda-out-of-memory-when-restart-from-checkpoint/217083/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 }62 ],63 "stream": [64 46616065 ]66 },67 "timeline_lookup": [68 [69 1,70 24371 ]72 ],73 "suggested_topics": [74 {75 "fancy_title": "How to handle training of few layers with DDP",76 "id": 219979,77 "title": "How to handle training of few layers with DDP",78 "slug": "how-to-handle-training-of-few-layers-with-ddp",79 "posts_count": 3,80 "reply_count": 1,81 "highest_post_number": 3,82 "image_url": null,83 "created_at": "2025-05-13T08:17:42.196Z",84 "last_posted_at": "2025-05-14T08:40:41.263Z",85 "bumped": true,86 "bumped_at": "2025-05-14T08:40:41.263Z",87 "archetype": "regular",88 "unseen": false,89 "pinned": false,90 "unpinned": null,91 "visible": true,92 "closed": false,93 "archived": false,94 "bookmarked": null,95 "liked": null,96 "tags_descriptions": {},97 "like_count": 0,98 "views": 62,99 "category_id": 12,100 "featured_link": null,101 "has_accepted_answer": true,102 "posters": [103 {104 "extras": "latest",105 "description": "Original Poster, Most Recent Poster",106 "user": {107 "id": 21407,108 "username": "Guillaume_Jeanneret",109 "name": "Guillaume Jeanneret",110 "avatar_template": "/user_avatar/discuss.pytorch.org/guillaume_jeanneret/{size}/14837_2.png",111 "trust_level": 1112 }113 },114 {115 "extras": null,116 "description": "Frequent Poster, Accepted Answer",117 "user": {118 "id": 3534,119 "username": "ptrblck",120 "name": "",121 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",122 "admin": true,123 "moderator": true,124 "trust_level": 2125 }126 }127 ]128 },129 {130 "fancy_title": "Socket error - broken pipe during rendezvous",131 "id": 220550,132 "title": "Socket error - broken pipe during rendezvous",133 "slug": "socket-error-broken-pipe-during-rendezvous",134 "posts_count": 8,135 "reply_count": 5,136 "highest_post_number": 8,137 "image_url": null,138 "created_at": "2025-06-03T18:58:36.006Z",139 "last_posted_at": "2025-06-24T18:47:55.909Z",140 "bumped": true,141 "bumped_at": "2025-06-24T18:47:55.909Z",142 "archetype": "regular",143 "unseen": false,144 "pinned": false,145 "unpinned": null,146 "visible": true,147 "closed": false,148 "archived": false,149 "bookmarked": null,150 "liked": null,151 "tags_descriptions": {},152 "like_count": 0,153 "views": 290,154 "category_id": 12,155 "featured_link": null,156 "has_accepted_answer": false,157 "posters": [158 {159 "extras": null,160 "description": "Original Poster",161 "user": {162 "id": 84566,163 "username": "Matt_Rae",164 "name": "Matt Rae",165 "avatar_template": "/user_avatar/discuss.pytorch.org/matt_rae/{size}/74760_2.png",166 "trust_level": 0167 }168 },169 {170 "extras": null,171 "description": "Frequent Poster",172 "user": {173 "id": 3534,174 "username": "ptrblck",175 "name": "",176 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",177 "admin": true,178 "moderator": true,179 "trust_level": 2180 }181 },182 {183 "extras": null,184 "description": "Frequent Poster",185 "user": {186 "id": 54320,187 "username": "fduwjj",188 "name": "Hugo",189 "avatar_template": "/user_avatar/discuss.pytorch.org/fduwjj/{size}/47855_2.png",190 "trust_level": 2191 }192 },193 {194 "extras": "latest",195 "description": "Most Recent Poster",196 "user": {197 "id": 84815,198 "username": "julian-q",199 "name": "Julian Quevedo",200 "avatar_template": "/user_avatar/discuss.pytorch.org/julian-q/{size}/77469_2.png",201 "trust_level": 0202 }203 }204 ]205 },206 {207 "fancy_title": "Memory error on ONE GPU destribution on the CPU befor moving the data",208 "id": 219118,209 "title": "Memory error on ONE GPU destribution on the CPU befor moving the data",210 "slug": "memory-error-on-one-gpu-destribution-on-the-cpu-befor-moving-the-data",211 "posts_count": 4,212 "reply_count": 2,213 "highest_post_number": 4,214 "image_url": null,215 "created_at": "2025-04-15T17:54:47.025Z",216 "last_posted_at": "2025-04-15T23:18:33.367Z",217 "bumped": true,218 "bumped_at": "2025-04-15T23:18:33.367Z",219 "archetype": "regular",220 "unseen": false,221 "pinned": false,222 "unpinned": null,223 "visible": true,224 "closed": false,225 "archived": false,226 "bookmarked": null,227 "liked": null,228 "tags_descriptions": {},229 "like_count": 0,230 "views": 61,231 "category_id": 12,232 "featured_link": null,233 "has_accepted_answer": false,234 "posters": [235 {236 "extras": null,237 "description": "Original Poster",238 "user": {239 "id": 82744,240 "username": "Maria_Djeblahi",241 "name": "Maria Djeblahi",242 "avatar_template": "/user_avatar/discuss.pytorch.org/maria_djeblahi/{size}/75704_2.png",243 "trust_level": 1244 }245 },246 {247 "extras": "latest",248 "description": "Most Recent Poster",249 "user": {250 "id": 3534,251 "username": "ptrblck",252 "name": "",253 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",254 "admin": true,255 "moderator": true,256 "trust_level": 2257 }258 }259 ]260 },261 {262 "fancy_title": "Issue With Forward Hooks in Deterministic Multi-GPU training",263 "id": 212557,264 "title": "Issue With Forward Hooks in Deterministic Multi-GPU training",265 "slug": "issue-with-forward-hooks-in-deterministic-multi-gpu-training",266 "posts_count": 1,267 "reply_count": 0,268 "highest_post_number": 1,269 "image_url": null,270 "created_at": "2024-11-05T15:18:44.275Z",271 "last_posted_at": "2024-11-05T15:18:44.347Z",272 "bumped": true,273 "bumped_at": "2024-11-05T15:18:44.347Z",274 "archetype": "regular",275 "unseen": false,276 "pinned": false,277 "unpinned": null,278 "visible": true,279 "closed": false,280 "archived": false,281 "bookmarked": null,282 "liked": null,283 "tags_descriptions": {},284 "like_count": 0,285 "views": 36,286 "category_id": 12,287 "featured_link": null,288 "has_accepted_answer": false,289 "posters": [290 {291 "extras": "latest single",292 "description": "Original Poster, Most Recent Poster",293 "user": {294 "id": 78912,295 "username": "Tanay_Arora",296 "name": "Tanay Arora",297 "avatar_template": "/user_avatar/discuss.pytorch.org/tanay_arora/{size}/72747_2.png",298 "trust_level": 1299 }300 }301 ]302 },303 {304 "fancy_title": "Copying params between 2 identically sharded (FSDP) networks",305 "id": 220944,306 "title": "Copying params between 2 identically sharded (FSDP) networks",307 "slug": "copying-params-between-2-identically-sharded-fsdp-networks",308 "posts_count": 3,309 "reply_count": 1,310 "highest_post_number": 3,311 "image_url": null,312 "created_at": "2025-06-20T11:02:43.179Z",313 "last_posted_at": "2025-07-16T14:00:29.979Z",314 "bumped": true,315 "bumped_at": "2025-07-16T14:00:29.979Z",316 "archetype": "regular",317 "unseen": false,318 "pinned": false,319 "unpinned": null,320 "visible": true,321 "closed": false,322 "archived": false,323 "bookmarked": null,324 "liked": null,325 "tags_descriptions": {},326 "like_count": 0,327 "views": 84,328 "category_id": 12,329 "featured_link": null,330 "has_accepted_answer": false,331 "posters": [332 {333 "extras": "latest",334 "description": "Original Poster, Most Recent Poster",335 "user": {336 "id": 84767,337 "username": "Timofey_Abramski",338 "name": "Timofey Abramski",339 "avatar_template": "/user_avatar/discuss.pytorch.org/timofey_abramski/{size}/72700_2.png",340 "trust_level": 0341 }342 },343 {344 "extras": null,345 "description": "Frequent Poster",346 "user": {347 "id": 84995,348 "username": "bantmen",349 "name": "",350 "avatar_template": "/user_avatar/discuss.pytorch.org/bantmen/{size}/77624_2.png",351 "trust_level": 1352 }353 }354 ]355 }356 ],357 "tags_descriptions": {},358 "fancy_title": "Cuda out of memory when restart from checkpoint",359 "id": 217083,360 "title": "Cuda out of memory when restart from checkpoint",361 "posts_count": 1,362 "created_at": "2025-02-24T07:26:18.746Z",363 "views": 68,364 "reply_count": 0,365 "like_count": 0,366 "last_posted_at": "2025-02-24T07:26:18.787Z",367 "visible": true,368 "closed": false,369 "archived": false,370 "has_summary": false,371 "archetype": "regular",372 "slug": "cuda-out-of-memory-when-restart-from-checkpoint",373 "category_id": 12,374 "word_count": 105,375 "deleted_at": null,376 "user_id": 82890,377 "featured_link": null,378 "pinned_globally": false,379 "pinned_at": null,380 "pinned_until": null,381 "image_url": null,382 "slow_mode_seconds": 0,383 "draft": null,384 "draft_key": "topic_217083",385 "draft_sequence": null,386 "unpinned": null,387 "pinned": false,388 "current_post_number": 1,389 "highest_post_number": 1,390 "deleted_by": null,391 "actions_summary": [392 {393 "id": 4,394 "count": 0,395 "hidden": false,396 "can_act": false397 },398 {399 "id": 8,400 "count": 0,401 "hidden": false,402 "can_act": false403 },404 {405 "id": 10,406 "count": 0,407 "hidden": false,408 "can_act": false409 },410 {411 "id": 7,412 "count": 0,413 "hidden": false,414 "can_act": false415 }416 ],417 "chunk_size": 20,418 "bookmarked": false,419 "topic_timer": null,420 "message_bus_last_id": 0,421 "participant_count": 1,422 "show_read_indicator": false,423 "thumbnails": null,424 "slow_mode_enabled_until": null,425 "can_vote": false,426 "vote_count": 0,427 "user_voted": false,428 "discourse_zendesk_plugin_zendesk_id": null,429 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",430 "details": {431 "can_edit": false,432 "notification_level": 1,433 "participants": [434 {435 "id": 82890,436 "username": "Chris_YU",437 "name": "Chris YU",438 "avatar_template": "/user_avatar/discuss.pytorch.org/chris_yu/{size}/75844_2.png",439 "post_count": 1,440 "primary_group_name": null,441 "flair_name": null,442 "flair_url": null,443 "flair_color": null,444 "flair_bg_color": null,445 "flair_group_id": null,446 "trust_level": 1447 }448 ],449 "created_by": {450 "id": 82890,451 "username": "Chris_YU",452 "name": "Chris YU",453 "avatar_template": "/user_avatar/discuss.pytorch.org/chris_yu/{size}/75844_2.png"454 },455 "last_poster": {456 "id": 82890,457 "username": "Chris_YU",458 "name": "Chris YU",459 "avatar_template": "/user_avatar/discuss.pytorch.org/chris_yu/{size}/75844_2.png"460 }461 },462 "bookmarks": []463 },464 {465 "post_stream": {466 "posts": [467 {468 "id": 466123,469 "name": "amir T mgr",470 "username": "amirtmgr",471 "avatar_template": "/user_avatar/discuss.pytorch.org/amirtmgr/{size}/49195_2.png",472 "created_at": "2025-02-23T16:30:46.009Z",473 "cooked": "<p>I came across this paper called, <a href=\"https://www.researchgate.net/publication/350757416_Boosting_Inertial-Based_Human_Activity_Recognition_With_Transformers\" rel=\"noopener nofollow ugc\">Inertial-based Activity Recognition with Transformers</a>, where authors have utilized CNN + Transformer encoders + FCs to classify human activities from sensor data.</p>\n<p>I deep-dived into the code implementation and found they are using Convolutional layers of kernel size 1x1. Can anyone explain why the author chose 1x1 kernel-size convolutional layers? What are the significance or effects of these in the model?</p>\n<p>To my understanding, the Conv layers with 1x1 kernel size generate new high-level features incorporating data from all channels without changing the step sizes.</p>\n<p>Am I right here? Can someone explain the significance or effects of using 1x1 kernel size in such a scenario of human activity recognition from sensor data?</p>",474 "post_number": 1,475 "post_type": 1,476 "posts_count": 2,477 "updated_at": "2025-02-23T16:30:46.009Z",478 "reply_count": 0,479 "reply_to_post_number": null,480 "quote_count": 0,481 "incoming_link_count": 10,482 "reads": 4,483 "readers_count": 3,484 "score": 50.8,485 "yours": false,486 "topic_id": 217058,487 "topic_slug": "reasons-behind-using-convolutional-layers-of-1x1-kernel-sizes",488 "display_username": "amir T mgr",489 "primary_group_name": null,490 "flair_name": null,491 "flair_url": null,492 "flair_bg_color": null,493 "flair_color": null,494 "flair_group_id": null,495 "badges_granted": [],496 "version": 1,497 "can_edit": false,498 "can_delete": false,499 "can_recover": false,500 "can_see_hidden_post": false,501 "can_wiki": false,502 "link_counts": [503 {504 "url": "https://www.researchgate.net/publication/350757416_Boosting_Inertial-Based_Human_Activity_Recognition_With_Transformers",505 "internal": false,506 "reflection": false,507 "clicks": 0508 }509 ],510 "read": true,511 "user_title": null,512 "bookmarked": false,513 "actions_summary": [],514 "moderator": false,515 "admin": false,516 "staff": false,517 "user_id": 55555,518 "hidden": false,519 "trust_level": 1,520 "deleted_at": null,521 "user_deleted": false,522 "edit_reason": null,523 "can_view_edit_history": true,524 "wiki": false,525 "post_url": "/t/reasons-behind-using-convolutional-layers-of-1x1-kernel-sizes/217058/1",526 "can_accept_answer": false,527 "can_unaccept_answer": false,528 "accepted_answer": false,529 "topic_accepted_answer": null,530 "can_vote": false531 },532 {533 "id": 466150,534 "name": "Anant Gupta",535 "username": "anantguptadbl",536 "avatar_template": "/user_avatar/discuss.pytorch.org/anantguptadbl/{size}/17784_2.png",537 "created_at": "2025-02-24T04:54:29.251Z",538 "cooked": "<p>Sounds correct to me. Best way is to visualize the output from a single image.<br>\nYou can try this in a jupyter notebook</p>\n<pre><code class=\"lang-auto\">from sklearn.datasets import load_sample_image\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n%matplotlib inline\nflower = load_sample_image('flower.jpg') \nprint(flower.shape)\nplt.imshow(flower)\nplt.figure()\n\nflower_torch = torch.from_numpy(flower.transpose(2,0,1).astype(np.float32))\n\nconv_layer = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=1, stride=1, padding=0)\noutput = conv_layer(flower_torch)\nprint(output.detach().numpy().shape)\nplt.imshow(output.detach().numpy().transpose(1,2,0))\n</code></pre>",539 "post_number": 2,540 "post_type": 1,541 "posts_count": 2,542 "updated_at": "2025-02-24T04:54:29.251Z",543 "reply_count": 0,544 "reply_to_post_number": null,545 "quote_count": 0,546 "incoming_link_count": 0,547 "reads": 3,548 "readers_count": 2,549 "score": 0.6,550 "yours": false,551 "topic_id": 217058,552 "topic_slug": "reasons-behind-using-convolutional-layers-of-1x1-kernel-sizes",553 "display_username": "Anant Gupta",554 "primary_group_name": null,555 "flair_name": null,556 "flair_url": null,557 "flair_bg_color": null,558 "flair_color": null,559 "flair_group_id": null,560 "badges_granted": [],561 "version": 1,562 "can_edit": false,563 "can_delete": false,564 "can_recover": false,565 "can_see_hidden_post": false,566 "can_wiki": false,567 "read": true,568 "user_title": "",569 "bookmarked": false,570 "actions_summary": [],571 "moderator": false,572 "admin": false,573 "staff": false,574 "user_id": 19553,575 "hidden": false,576 "trust_level": 2,577 "deleted_at": null,578 "user_deleted": false,579 "edit_reason": null,580 "can_view_edit_history": true,581 "wiki": false,582 "post_url": "/t/reasons-behind-using-convolutional-layers-of-1x1-kernel-sizes/217058/2",583 "can_accept_answer": false,584 "can_unaccept_answer": false,585 "accepted_answer": false,586 "topic_accepted_answer": null587 }588 ],589 "stream": [590 466123,591 466150592 ]593 },594 "timeline_lookup": [595 [596 1,597 244598 ]599 ],600 "suggested_topics": [601 {602 "fancy_title": "How to share a model in multi process?",603 "id": 212232,604 "title": "How to share a model in multi process?",605 "slug": "how-to-share-a-model-in-multi-process",606 "posts_count": 1,607 "reply_count": 0,608 "highest_post_number": 1,609 "image_url": null,610 "created_at": "2024-10-29T02:34:33.270Z",611 "last_posted_at": "2024-10-29T02:34:33.328Z",612 "bumped": true,613 "bumped_at": "2024-10-29T02:34:33.328Z",614 "archetype": "regular",615 "unseen": false,616 "pinned": false,617 "unpinned": null,618 "visible": true,619 "closed": false,620 "archived": false,621 "bookmarked": null,622 "liked": null,623 "tags_descriptions": {},624 "like_count": 0,625 "views": 23,626 "category_id": 1,627 "featured_link": null,628 "has_accepted_answer": false,629 "posters": [630 {631 "extras": "latest single",632 "description": "Original Poster, Most Recent Poster",633 "user": {634 "id": 79055,635 "username": "hello_e",636 "name": "hhhh",637 "avatar_template": "/letter_avatar_proxy/v4/letter/h/f9ae1b/{size}.png",638 "trust_level": 1639 }640 }641 ]642 },643 {644 "fancy_title": "Couldn’t find the retrieval_recall in torcheval",645 "id": 217950,646 "title": "Couldn't find the retrieval_recall in torcheval",647 "slug": "couldnt-find-the-retrieval-recall-in-torcheval",648 "posts_count": 3,649 "reply_count": 1,650 "highest_post_number": 3,651 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/0/c/0c90944d4d2e1934cd171736ed8ce0d1c93ffa37_2_1024x141.png",652 "created_at": "2025-03-17T12:32:04.660Z",653 "last_posted_at": "2025-03-18T01:21:23.396Z",654 "bumped": true,655 "bumped_at": "2025-03-18T01:21:23.396Z",656 "archetype": "regular",657 "unseen": false,658 "pinned": false,659 "unpinned": null,660 "visible": true,661 "closed": false,662 "archived": false,663 "bookmarked": null,664 "liked": null,665 "tags_descriptions": {},666 "like_count": 1,667 "views": 53,668 "category_id": 1,669 "featured_link": null,670 "has_accepted_answer": true,671 "posters": [672 {673 "extras": "latest",674 "description": "Original Poster, Most Recent Poster",675 "user": {676 "id": 72736,677 "username": "songsong0425",678 "name": "Songyeon Lee",679 "avatar_template": "/user_avatar/discuss.pytorch.org/songsong0425/{size}/67200_2.png",680 "trust_level": 1681 }682 },683 {684 "extras": null,685 "description": "Frequent Poster, Accepted Answer",686 "user": {687 "id": 3534,688 "username": "ptrblck",689 "name": "",690 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",691 "admin": true,692 "moderator": true,693 "trust_level": 2694 }695 }696 ]697 },698 {699 "fancy_title": "nn.MultiheadAttention output",700 "id": 217358,701 "title": "nn.MultiheadAttention output",702 "slug": "nn-multiheadattention-output",703 "posts_count": 1,704 "reply_count": 0,705 "highest_post_number": 1,706 "image_url": null,707 "created_at": "2025-03-03T00:01:48.614Z",708 "last_posted_at": "2025-03-03T00:01:48.654Z",709 "bumped": true,710 "bumped_at": "2025-03-03T00:01:48.654Z",711 "archetype": "regular",712 "unseen": false,713 "pinned": false,714 "unpinned": null,715 "visible": true,716 "closed": false,717 "archived": false,718 "bookmarked": null,719 "liked": null,720 "tags_descriptions": {},721 "like_count": 0,722 "views": 46,723 "category_id": 1,724 "featured_link": null,725 "has_accepted_answer": false,726 "posters": [727 {728 "extras": "latest single",729 "description": "Original Poster, Most Recent Poster",730 "user": {731 "id": 83030,732 "username": "saimandahal",733 "name": "Saiman Dahal",734 "avatar_template": "/user_avatar/discuss.pytorch.org/saimandahal/{size}/75951_2.png",735 "trust_level": 1736 }737 }738 ]739 },740 {741 "fancy_title": "Why data loading performance degradation between Pytorch 1.12 and 2.6.0?",742 "id": 217564,743 "title": "Why data loading performance degradation between Pytorch 1.12 and 2.6.0?",744 "slug": "why-data-loading-performance-degradation-between-pytorch-1-12-and-2-6-0",745 "posts_count": 3,746 "reply_count": 1,747 "highest_post_number": 3,748 "image_url": null,749 "created_at": "2025-03-07T14:57:47.834Z",750 "last_posted_at": "2025-03-08T15:04:02.300Z",751 "bumped": true,752 "bumped_at": "2025-03-08T15:13:53.580Z",753 "archetype": "regular",754 "unseen": false,755 "pinned": false,756 "unpinned": null,757 "visible": true,758 "closed": false,759 "archived": false,760 "bookmarked": null,761 "liked": null,762 "tags_descriptions": {},763 "like_count": 0,764 "views": 148,765 "category_id": 1,766 "featured_link": null,767 "has_accepted_answer": true,768 "posters": [769 {770 "extras": "latest single",771 "description": "Original Poster, Most Recent Poster, Accepted Answer",772 "user": {773 "id": 69167,774 "username": "lostalot",775 "name": "",776 "avatar_template": "/letter_avatar_proxy/v4/letter/l/858c86/{size}.png",777 "trust_level": 1778 }779 }780 ]781 },782 {783 "fancy_title": "Flex_attention inference with variable length inputs",784 "id": 218250,785 "title": "Flex_attention inference with variable length inputs",786 "slug": "flex-attention-inference-with-variable-length-inputs",787 "posts_count": 2,788 "reply_count": 0,789 "highest_post_number": 2,790 "image_url": null,791 "created_at": "2025-03-25T13:59:33.953Z",792 "last_posted_at": "2025-03-29T22:38:41.920Z",793 "bumped": true,794 "bumped_at": "2025-03-29T22:38:41.920Z",795 "archetype": "regular",796 "unseen": false,797 "pinned": false,798 "unpinned": null,799 "visible": true,800 "closed": false,801 "archived": false,802 "bookmarked": null,803 "liked": null,804 "tags_descriptions": {},805 "like_count": 1,806 "views": 298,807 "category_id": 1,808 "featured_link": null,809 "has_accepted_answer": false,810 "posters": [811 {812 "extras": null,813 "description": "Original Poster",814 "user": {815 "id": 83407,816 "username": "jjjj",817 "name": null,818 "avatar_template": "/user_avatar/discuss.pytorch.org/jjjj/{size}/76279_2.png",819 "trust_level": 1820 }821 },822 {823 "extras": "latest",824 "description": "Most Recent Poster",825 "user": {826 "id": 78199,827 "username": "windsornguyen",828 "name": "Windsor Nguyễn",829 "avatar_template": "/user_avatar/discuss.pytorch.org/windsornguyen/{size}/72103_2.png",830 "trust_level": 1831 }832 }833 ]834 }835 ],836 "tags_descriptions": {},837 "fancy_title": "Reasons behind using convolutional layers of 1x1 kernel sizes",838 "id": 217058,839 "title": "Reasons behind using convolutional layers of 1x1 kernel sizes",840 "posts_count": 2,841 "created_at": "2025-02-23T16:30:45.966Z",842 "views": 64,843 "reply_count": 0,844 "like_count": 0,845 "last_posted_at": "2025-02-24T04:54:29.251Z",846 "visible": true,847 "closed": false,848 "archived": false,849 "has_summary": false,850 "archetype": "regular",851 "slug": "reasons-behind-using-convolutional-layers-of-1x1-kernel-sizes",852 "category_id": 1,853 "word_count": 219,854 "deleted_at": null,855 "user_id": 55555,856 "featured_link": null,857 "pinned_globally": false,858 "pinned_at": null,859 "pinned_until": null,860 "image_url": null,861 "slow_mode_seconds": 0,862 "draft": null,863 "draft_key": "topic_217058",864 "draft_sequence": null,865 "unpinned": null,866 "pinned": false,867 "current_post_number": 1,868 "highest_post_number": 2,869 "deleted_by": null,870 "actions_summary": [871 {872 "id": 4,873 "count": 0,874 "hidden": false,875 "can_act": false876 },877 {878 "id": 8,879 "count": 0,880 "hidden": false,881 "can_act": false882 },883 {884 "id": 10,885 "count": 0,886 "hidden": false,887 "can_act": false888 },889 {890 "id": 7,891 "count": 0,892 "hidden": false,893 "can_act": false894 }895 ],896 "chunk_size": 20,897 "bookmarked": false,898 "topic_timer": null,899 "message_bus_last_id": 0,900 "participant_count": 2,901 "show_read_indicator": false,902 "thumbnails": null,903 "slow_mode_enabled_until": null,904 "can_vote": false,905 "vote_count": 0,906 "user_voted": false,907 "discourse_zendesk_plugin_zendesk_id": null,908 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",909 "details": {910 "can_edit": false,911 "notification_level": 1,912 "participants": [913 {914 "id": 19553,915 "username": "anantguptadbl",916 "name": "Anant Gupta",917 "avatar_template": "/user_avatar/discuss.pytorch.org/anantguptadbl/{size}/17784_2.png",918 "post_count": 1,919 "primary_group_name": null,920 "flair_name": null,921 "flair_url": null,922 "flair_color": null,923 "flair_bg_color": null,924 "flair_group_id": null,925 "trust_level": 2926 },927 {928 "id": 55555,929 "username": "amirtmgr",930 "name": "amir T mgr",931 "avatar_template": "/user_avatar/discuss.pytorch.org/amirtmgr/{size}/49195_2.png",932 "post_count": 1,933 "primary_group_name": null,934 "flair_name": null,935 "flair_url": null,936 "flair_color": null,937 "flair_bg_color": null,938 "flair_group_id": null,939 "trust_level": 1940 }941 ],942 "created_by": {943 "id": 55555,944 "username": "amirtmgr",945 "name": "amir T mgr",946 "avatar_template": "/user_avatar/discuss.pytorch.org/amirtmgr/{size}/49195_2.png"947 },948 "last_poster": {949 "id": 19553,950 "username": "anantguptadbl",951 "name": "Anant Gupta",952 "avatar_template": "/user_avatar/discuss.pytorch.org/anantguptadbl/{size}/17784_2.png"953 }954 },955 "bookmarks": []956 },957 {958 "post_stream": {959 "posts": [960 {961 "id": 466148,962 "name": "Minh Tu",963 "username": "tcm",964 "avatar_template": "/user_avatar/discuss.pytorch.org/tcm/{size}/74956_2.png",965 "created_at": "2025-02-24T04:51:05.484Z",966 "cooked": "<p>Hi,</p>\n<p>I’m finetuning a multimodal LLM and during this process, I encounter the following error when attempting to save the checkpoint. More particularly, I can save the model normally but when the optimizer states are saved, the following error occurs:</p>\n<pre><code class=\"lang-auto\">RuntimeError: Detected mismatch between collectives on ranks. Rank 0 is running collective: CollectiveFingerPrint(SequenceNumber=549, OpType=ALLGATHER, TensorShape=[0], TensorDtypes=Float, TensorDeviceTypes=TensorOptions(dtype=float (default), device=cuda, layout=Strided (default), requires_grad=false (default), pinned_memory=false (default), memory_format=(nullopt))), but Rank 2 is running collective: CollectiveFingerPrint(SequenceNumber=549, OpType=ALLGATHER, TensorShape=[183971584], TensorDtypes=Float, TensorDeviceTypes=TensorOptions(dtype=float (default), device=cuda, layout=Strided (default), requires_grad=false (default), pinned_memory=false (default), memory_format=(nullopt))).Collectives differ in the following aspects: Tensor Tensor shapes: 0vs 183971584\n</code></pre>\n<p>This is the full traceback of tensor shape mismatch when saving fsdp optimizer states:</p>\n<pre><code class=\"lang-auto\">File \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/transformers/trainer.py\", line 2356, in _inner_training_loop\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/transformers/trainer.py\", line 2807, in _maybe_log_save_evaluate\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/transformers/trainer.py\", line 2890, in _save_checkpoint\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/transformers/trainer.py\", line 3001, in _save_optimizer_and_scheduler\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/accelerate/utils/fsdp_utils.py\", line 185, in save_fsdp_optimizer\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py\", line 1828, in optim_state_dict\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py\", line 1253, in _optim_state_dict_impl\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/torch/distributed/fsdp/_optim_utils.py\", line 1396, in _optim_state_dict\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/torch/distributed/fsdp/_optim_utils.py\", line 1657, in _gather_orig_param_state\nFile \"/media02/nthuy/miniconda3/envs/thesis_longvu/lib/python3.10/site-packages/torch/distributed/fsdp/_optim_utils.py\", line 1593, in _all_gather_optim_state\n</code></pre>\n<p>The packages version is:</p>\n<pre><code class=\"lang-auto\">torch==2.1.2\nnumpy==1.26.4\ntransformers==4.43.1\n</code></pre>\n<p>This is the full details I trace the error in functions listed in the traceback. It’s quite long but it leads to what I believe a possible reason for error above and my question below:</p>\n<details>\n<summary>\nSummary</summary>\n<p>I began digging into the codebase, starting with <code>torch.distributed.fsdp</code>, to find out the cause as follows:</p>\n<ul>\n<li>First of all, the error stems from this function: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1513\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\nIt happens that when execution enters this line</li>\n</ul>\n<pre><code class=\"lang-auto\">work = dist.all_gather(\n tensors, local_state, group=fsdp_state.process_group, async_op=True\n )\n</code></pre>\n<p><code>tensors</code> is a list where certain elements are 0 and others are non-zero, as raised by the error above. When I printed out <code>object_state.tensors</code>:</p>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:42,065 - root - DEBUG - rank 0, object_state.tensors: {}, name: exp_avg, info: None\n2025-02-22 17:02:42,065 - root - DEBUG - rank 1, object_state.tensors: {}, name: exp_avg, info: None\n2025-02-22 17:02:42,066 - root - DEBUG - rank 2, object_state.tensors: {'exp_avg': _PosDimTensorInfo(shape=torch.Size([183971584]), dtype=torch.float32), 'exp_avg_sq': _PosDimTensorInfo(shape=torch.Size([183971584]), dtype=torch.float32)}, name: exp_avg, info: _PosDimTensorInfo(shape=torch.Size([183971584]), dtype=torch.float32) \n2025-02-22 17:02:42,066 - root - DEBUG - rank 3, object_state.tensors: {'exp_avg': _PosDimTensorInfo(shape=torch.Size([210030848]), dtype=torch.float32), 'exp_avg_sq': _PosDimTensorInfo(shape=torch.Size([210030848]), dtype=torch.float32)}, name: exp_avg, info: _PosDimTensorInfo(shape=torch.Size([210030848]), dtype=torch.float32)\n</code></pre>\n<p>It can be seen that on the 2 ranks 0 and 1, the tensors are empty. Since these <code>object_state.tensors</code> are gathered in <code>object_list</code> from processes in the process group via:</p>\n<pre><code class=\"lang-auto\">dist.all_gather_object(object_list, processed_state, group=fsdp_state.process_group)\n</code></pre>\n<p>It seems that the <code>processed_state</code> for ranks 0 and 1 is empty (<code>StateInfo({}, {}, {})</code>). And this is caused by empty <code>optim_state</code> if you look at the for loop in the beginning of <code>_all_gather_optim_state()</code>.</p>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:42,060 - root - DEBUG - @tcm: In _all_gather_optim_state(): optim_state: {}\n</code></pre>\n<ul>\n<li>After that, I look into the second function in the traceback: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1611\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\nThe <code>optim_state</code> dictionary in this function is still empty.</li>\n</ul>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:42,060 - root - DEBUG - @tcm: In _gather_orig_param_state(): optim_state: {}\n</code></pre>\n<ul>\n<li>I continued looking into the next function in the traceback: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1269\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\nIn this function, I focused on the following part:</li>\n</ul>\n<pre><code class=\"lang-auto\"># Iterate in rank 0's flat parameter ID order to ensure aligned all-gathers\n # across ranks\n for optim_state_key in all_optim_state_keys:\n param_key: Union[str, int, None] = optim_state_key_to_param_key.get(\n optim_state_key, None\n )\n\n if param_key is None:\n assert use_orig_params, (\n \"If use_orig_params is False, we must be able to find the \"\n f\"corresponding param id. {optim_state_key} {param_key}\"\n )\n if not optim_state_key.is_fsdp_managed:\n continue\n\n if optim_state_key.is_fsdp_managed:\n # If there are multiple unflat_param_names (not use_orig_params),\n # they share the same FSDPParamInfo. So the first unflat_param_name\n # is sufficient to fetch the FSDPParamInfo.\n fqn = optim_state_key.unflat_param_names[0]\n fsdp_param_info = fqn_to_fsdp_param_info[fqn]\n if use_orig_params:\n state = (\n {} if param_key is None else optim_state_dict[\"state\"][param_key]\n )\n unflat_state = [\n _gather_orig_param_state(\n fsdp_param_info,\n fqn,\n state,\n shard_state,\n )\n ]\n</code></pre>\n<p>The problem is that <code>param_key</code> is <code>None</code> which leads to empty ‘state’ when passed to <code>_gather_orig_param_state()</code>:</p>\n<pre><code class=\"lang-auto\">state = ({} if param_key is None else optim_state_dict[\"state\"][param_key])\n</code></pre>\n<p>The <code>param_key</code> is <code>None</code> because <code>optim_state_key_to_param_key</code> dictionary is empty:</p>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:42,046 - root - DEBUG - @tcm: In _optim_state_dict(): optim_state_key_to_param_key: {} # rank 0 or 1\n2025-02-22 17:02:42,046 - root - DEBUG - @tcm: In _optim_state_dict(): optim_state_key_to_param_key: {_OptimStateKey(unflat_param_names=('lm_head.weight',), is_fsdp_managed=True): 2} # rank 2 or 3\n2025-02-22 17:02:42,046 - root - DEBUG - @tcm: In _optim_state_dict(): optim_state_key_to_param_key: {_OptimStateKey(unflat_param_names=('model.mm_projector.0.weight',), is_fsdp_managed=True): 0, _OptimStateKey(unflat_param_names=('model.mm_projector.2.weight',), is_fsdp_managed=True): 1, _OptimStateKey(unflat_param_names=('model.mm_projector.0.bias',), is_fsdp_managed=True): 3, _OptimStateKey(unflat_param_names=('model.mm_projector.2.bias',), is_fsdp_managed=True): 4} # rank 2 or 3\n</code></pre>\n<p>To understand why the dict <code>optim_state_key_to_param_key</code> is empty, I looked into the function: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1161\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\nHere, if we look at the for loop in the beginning:</p>\n<pre><code class=\"lang-auto\">for param_key, param in param_key_to_param.items():\n # Do not include parameters without state to avoid empty mappings\n # just like in normal `torch.optim.Optimizer.state_dict()`\n if param_key not in optim_state_dict[\"state\"]:\n continue\n</code></pre>\n<p><code>optim_state_dict[\"state\"]</code> is empty so the iteration is skipped, causing <code>optim_state_key_to_param_key</code> to not be updated.</p>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:42,041 - root - DEBUG - @tcm: In _map_param_key_to_optim_keys(): optim_state_dict[\"state\"]: {} # in empty ranks such as 0\n2025-02-22 17:02:28,436 - root - DEBUG - @tcm: In _map_param_key_to_optim_keys(): optim_state_dict[\"state\"]: {0: {'step': tensor(1.), 'exp_avg': tensor([-6.2440e-07, 6.8065e-07, -2.2726e-06, ..., 3.1220e-07, 1.6088e-06, -1.5047e-07], device='cuda:1'), 'exp_avg_sq': tensor([3.8987e-14, 4.6328e-14, 5.1646e-13, ..., 9.7467e-15, 2.5882e-13, 2.2642e-15], device='cuda:1')}...\n</code></pre>\n<p>So the problem is <code>optim_state_dict</code> being empty when passed into <code>_optim_state_dict()</code>.</p>\n<ul>\n<li>If we go further in the traceback:<br>\n<code>_optim_state_dict_impl()</code>: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/fully_sharded_data_parallel.py#1209\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/fully_sharded_data_parallel.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\n<code>_optim_state_dict()</code>: <a href=\"https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/fully_sharded_data_parallel.py#1741\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/distributed/fsdp/fully_sharded_data_parallel.py at v2.1.2 · pytorch/pytorch · GitHub</a><br>\nWe see that <code>optim_state_dict</code> is initially <code>None</code>:</li>\n</ul>\n<pre><code class=\"lang-auto\">2025-02-22 17:02:28,304 - root - DEBUG - @tcm: In FSDP.optim_state_dict(): optim_state_dict: None\n</code></pre>\n<p>and initialized through:</p>\n<pre><code class=\"lang-auto\">if optim_state_dict is None:\n optim_state_dict = optim.state_dict()\n</code></pre>\n<ul>\n<li>Now, when I go to transformers <code>trainer.py</code> and look at the following function:<br>\n<a href=\"https://github.com/huggingface/transformers/blob/v4.43.1/src/transformers/trainer.py#L2969\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">transformers/src/transformers/trainer.py at v4.43.1 · huggingface/transformers · GitHub</a><br>\nwe can see that when saving the fsdp optimizer, no <code>optim_state_dict</code> is passed:</li>\n</ul>\n<pre><code class=\"lang-auto\">save_fsdp_optimizer(self.accelerator.state.fsdp_plugin, self.accelerator, self.optimizer, self.model, output_dir)\n</code></pre>\n<p>So in <code>torch.distributed.fsdp._optim_utils.py</code>, the following function is used to initialize <code>optim_state_dict</code>: <a href=\"https://github.com/pytorch/pytorch/blob/main/torch/optim/optimizer.py#L636\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">pytorch/torch/optim/optimizer.py at main · pytorch/pytorch · GitHub</a></p>\n</details>\n<p>To start with, this is the architecture of the model that I’m trying to fine-tune:</p>\n<pre><code class=\"lang-auto\">FullyShardedDataParallel(\n (_fsdp_wrapped_module): CambrianLlamaForCausalLM(\n (model): CambrianLlamaModel(\n (embed_tokens): Embedding(128256, 3072)\n (layers): ModuleList(\n (0-27): 28 x FullyShardedDataParallel(\n (_fsdp_wrapped_module): LlamaDecoderLayer(\n (self_attn): LlamaSdpaAttention(\n (q_proj): Linear(in_features=3072, out_features=3072, bias=False)\n (k_proj): Linear(in_features=3072, out_features=1024, bias=False)\n (v_proj): Linear(in_features=3072, out_features=1024, bias=False)\n (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n (rotary_emb): LlamaRotaryEmbedding()\n )\n (mlp): LlamaMLP(\n (gate_proj): Linear(in_features=3072, out_features=8192, bias=False)\n (up_proj): Linear(in_features=3072, out_features=8192, bias=False)\n (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n (act_fn): SiLU()\n )\n (input_layernorm): LlamaRMSNorm()\n (post_attention_layernorm): LlamaRMSNorm()\n )\n )\n )\n (norm): LlamaRMSNorm()\n (rotary_emb): LlamaRotaryEmbedding()\n (mm_projector): Sequential(\n (0): Linear(in_features=1024, out_features=3072, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=3072, out_features=3072, bias=True)\n )\n (mm_projector_aux_0): Sequential(\n (0): Linear(in_features=1152, out_features=1024, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=1024, out_features=1024, bias=True)\n (3): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n )\n (mm_projector_aux_1): Sequential(\n (0): Linear(in_features=1536, out_features=1024, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=1024, out_features=1024, bias=True)\n (3): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n )\n (vision_sampler_0): VisionTokenSampler(\n (layers): ModuleList(\n (0-2): 3 x VisionCrossAttentionLayer(\n (proj_context): Linear(in_features=1024, out_features=1024, bias=False)\n (proj_in): Linear(in_features=2048, out_features=1024, bias=False)\n (proj_out): MLP(\n (linear_1): Linear(in_features=1024, out_features=1024, bias=False)\n (act): GELU(approximate='none')\n (linear_2): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (cross_attn): MultiKVCrossAttention(\n (q_proj): Sequential(\n (0): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (k_proj_0): Sequential(\n (0): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (v_proj_0): Sequential(\n (0): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (k_proj_1): Sequential(\n (0): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (v_proj_1): Sequential(\n (0): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=1024, out_features=1024, bias=False)\n )\n (o_proj): Linear(in_features=1024, out_features=1024, bias=False)\n )\n )\n )\n )\n (lm_head): Linear(in_features=3072, out_features=128256, bias=False)\n )\n )\n)\n\n</code></pre>\n<p>There are two FSDP instances: the entire model and the <code>LlamaDecoderLayer</code> layer. In my fine-tuning script, this is how I configure FSDP options when used in <code>Trainer</code>:</p>\n<pre><code class=\"lang-auto\">--fsdp \"full_shard auto_wrap\" \\\n--fsdp_transformer_layer_cls_to_wrap 'LlamaDecoderLayer' \\\n</code></pre>\n<p>Since the error is related to saving FSDP optimizer states, I would like to provide the following method in <code>LLaVATrainer</code> class, which is a subclass from Huggingface <code>Trainer</code>:</p>\n<pre><code class=\"lang-auto\">class LLaVATrainer(Trainer):\n def create_optimizer(self):\n \"\"\"\n Setup the optimizer.\n\n We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the\n Trainer's init through `optimizers`, or subclass and override this method in a subclass.\n \"\"\"\n # pyre-fixme[16]: `Trainer` has no attribute `model`.\n opt_model = self.model\n # if self.args.unfreeze_mm_vision_tower:\n # opt_model.get_model().vision_tower_aux_list = nn.ModuleList(opt_model.get_vision_tower_aux_list())\n # self.param_to_name = map_params_to_module_names([opt_model])\n # pyre-fixme[16]: `Trainer` has no attribute `optimizer`.\n if self.optimizer is None:\n decay_parameters = get_parameter_names(opt_model, ALL_LAYERNORM_LAYERS)\n decay_parameters = [name for name in decay_parameters if \"bias\" not in name]\n # pyre-fixme[16]: `Trainer` has no attribute `mm_projector_lr`.\n assert not (self.args.mm_projector_lr and self.args.mm_vision_sampler_lr)\n optimizer_grouped_parameters = [\n {\n \"params\": [\n p\n for n, p in opt_model.named_parameters()\n if (n in decay_parameters and p.requires_grad)\n ],\n \"weight_decay\": self.args.weight_decay,\n },\n {\n \"params\": [\n p\n for n, p in opt_model.named_parameters()\n if (n not in decay_parameters and p.requires_grad)\n ],\n \"weight_decay\": 0.0,\n },\n ]\noptimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(\n self.args\n )\n\n self.optimizer = optimizer_cls(\n optimizer_grouped_parameters, **optimizer_kwargs\n )\n return self.optimizer\n</code></pre>\n<p>In the model codebase, the <code>create_optimizer()</code> method above creates param groups and other params. When I print out the steps in this <code>create_optimizer()</code> method, the output is as follows:</p>\n<pre><code class=\"lang-auto\">2025-02-22 17:01:20,658 - root - DEBUG - @tcm: In LLaVATrainer: n = _fsdp_wrapped_module.lm_head.weight, p = Parameter containing: tensor([], device='cuda:0', dtype=torch.bfloat16, requires_grad=True)\n2025-02-22 17:01:20,695 - root - DEBUG - @tcm: In LLaVATrainer: n = _fsdp_wrapped_module.lm_head.weight, p = Parameter containing:\ntensor([], device='cuda:1', dtype=torch.bfloat16, requires_grad=True)\n...\n2025-02-22 17:01:20,690 - root - DEBUG - @tcm: In LLaVATrainer: n = _fsdp_wrapped_module.lm_head.weight, p = Parameter containing: tensor([ 0.0103, 0.0090, 0.0134, ..., 0.0049, -0.0025, -0.0052], device='cuda:2', dtype=torch.bfloat16, requires_grad=True)\n2025-02-22 17:01:20,691 - root - DEBUG - @tcm: In LLaVATrainer: n = _fsdp_wrapped_module.lm_head.weight, p = Parameter containing: tensor([-0.0099, -0.0302, -0.0054, ..., -0.0038, -0.0027, -0.0015],\n device='cuda:3', dtype=torch.bfloat16, requires_grad=True)\n\n</code></pre>\n<p>As can be seen, on ranks 0 and 1, the <code>LM head</code> layer wrapped in FSDP unit has no params in the tensor, but they are present on ranks 3 and 4. So I think the error might stem from this fsdp sharding where the same layer <code>lm head</code> is sharded on ranks 2 and 3 but empty on ranks 0 and 1. Therefore, <strong>my question is</strong>:</p>\n<p>I would like to ask why FSDP in <code>Trainer</code> shards a layer such that it’s empty on certain ranks, possibly leading to the error above?</p>\n<ol start=\"2\">\n<li>I further investigated <code>optim.Optimizer.state_dict()</code>and noticed that for ranks 1, 2, 3, the <code>self.state</code> dict is available:</li>\n</ol>\n<pre><code class=\"lang-auto\">2025-02-24 04:16:07,667 - root - DEBUG - @tcm: In optim.Optimizer.state_dict(): self.state=defaultdict(<class 'dict'>, {Parameter containing: tensor([-0.0099, -0.0302, -0.0054, ..., -0.0038, -0.0027, -0.0015],\n device='cuda:3', requires_grad=True): {'step': tensor(1.), 'exp_avg': tensor([-2.1631e-11, -6.3862e-11, -3.7596e-11, ..., -1.6094e-12, 7.6394e-12, -6.6093e-12], device='cuda:3'), 'exp_avg_sq': tensor([4.6788e-23, 4.0783e-22, 1.4135e-22, ..., 2.5902e-25, 5.8360e-24, 4.3683e-24], device='cuda:3')}})\n2025-02-24 04:16:07,667 - root - DEBUG - @tcm: In optim.Optimizer.state_dict(): self.state=defaultdict(<class 'dict'>, {Parameter containing: tensor([ 0.0103, 0.0090, 0.0134, ..., 0.0049, -0.0025, -0.0052],\n device='cuda:2', requires_grad=True): {'step': tensor(1.), 'exp_avg': tensor([-1.9513e-08, 5.0979e-09, 1.5206e-08, ..., -2.0051e-10, 9.6136e-11, -1.6412e-10], device='cuda:2'), 'exp_avg_sq': tensor([3.8075e-17, 2.5989e-18, 2.3122e-17, ..., 4.0205e-21, 9.2421e-22, 2.6935e-21], device='cuda:2')}})\n</code></pre>\n<p>but in rank 0, the <code>self.state</code> dict is empty:</p>\n<pre><code class=\"lang-auto\">2025-02-24 04:16:18,691 - root - DEBUG - @tcm: In optim.Optimizer.state_dict(): self.state=defaultdict(<class 'dict'>, {})\n</code></pre>\n<p>So I think this is the root cause of the error I’m asking all along. I don’t understand why for rank 0, the optimizer’s <code>self.state</code> is empty unlike the other ranks.</p>\n<p>I’m been trying my best to find out the root cause and fix this error but the codebase is large and complex, so I am seeking help from the community.</p>\n<p>Thanks in advance.</p>",967 "post_number": 1,968 "post_type": 1,969 "posts_count": 1,970 "updated_at": "2025-02-24T04:51:05.484Z",971 "reply_count": 0,972 "reply_to_post_number": null,973 "quote_count": 0,974 "incoming_link_count": 115,975 "reads": 6,976 "readers_count": 5,977 "score": 576.2,978 "yours": false,979 "topic_id": 217074,980 "topic_slug": "tensor-shape-mismatch-error-when-doing-an-allgather-in-distributed-training-with-fsdp",981 "display_username": "Minh Tu",982 "primary_group_name": null,983 "flair_name": null,984 "flair_url": null,985 "flair_bg_color": null,986 "flair_color": null,987 "flair_group_id": null,988 "badges_granted": [],989 "version": 1,990 "can_edit": false,991 "can_delete": false,992 "can_recover": false,993 "can_see_hidden_post": false,994 "can_wiki": false,995 "link_counts": [996 {997 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1269",998 "internal": false,999 "reflection": false,1000 "title": "pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub",1001 "clicks": 11002 },1003 {1004 "url": "https://github.com/huggingface/transformers/blob/v4.43.1/src/transformers/trainer.py#L2969",1005 "internal": false,1006 "reflection": false,1007 "title": "transformers/src/transformers/trainer.py at v4.43.1 · huggingface/transformers · GitHub",1008 "clicks": 01009 },1010 {1011 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/fully_sharded_data_parallel.py#1209",1012 "internal": false,1013 "reflection": false,1014 "title": "pytorch/torch/distributed/fsdp/fully_sharded_data_parallel.py at v2.1.2 · pytorch/pytorch · GitHub",1015 "clicks": 01016 },1017 {1018 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1611",1019 "internal": false,1020 "reflection": false,1021 "title": "pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub",1022 "clicks": 01023 },1024 {1025 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1513",1026 "internal": false,1027 "reflection": false,1028 "title": "pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub",1029 "clicks": 01030 },1031 {1032 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/fully_sharded_data_parallel.py#1741",1033 "internal": false,1034 "reflection": false,1035 "title": "pytorch/torch/distributed/fsdp/fully_sharded_data_parallel.py at v2.1.2 · pytorch/pytorch · GitHub",1036 "clicks": 01037 },1038 {1039 "url": "https://github.com/pytorch/pytorch/blob/v2.1.2/torch/distributed/fsdp/_optim_utils.py#L1161",1040 "internal": false,1041 "reflection": false,1042 "title": "pytorch/torch/distributed/fsdp/_optim_utils.py at v2.1.2 · pytorch/pytorch · GitHub",1043 "clicks": 01044 },1045 {1046 "url": "https://github.com/pytorch/pytorch/blob/main/torch/optim/optimizer.py#L636",1047 "internal": false,1048 "reflection": false,1049 "title": "pytorch/torch/optim/optimizer.py at main · pytorch/pytorch · GitHub",1050 "clicks": 01051 }1052 ],1053 "read": true,1054 "user_title": null,1055 "bookmarked": false,1056 "actions_summary": [],1057 "moderator": false,1058 "admin": false,1059 "staff": false,1060 "user_id": 81933,1061 "hidden": false,1062 "trust_level": 1,1063 "deleted_at": null,1064 "user_deleted": false,1065 "edit_reason": null,1066 "can_view_edit_history": true,1067 "wiki": false,1068 "post_url": "/t/tensor-shape-mismatch-error-when-doing-an-allgather-in-distributed-training-with-fsdp/217074/1",1069 "can_accept_answer": false,1070 "can_unaccept_answer": false,1071 "accepted_answer": false,1072 "topic_accepted_answer": null,1073 "can_vote": false1074 }1075 ],1076 "stream": [1077 4661481078 ]1079 },1080 "timeline_lookup": [1081 [1082 1,1083 2441084 ]1085 ],1086 "suggested_topics": [1087 {1088 "fancy_title": "DistNetworkError when using multiprocessing_context parameter in pytorch dataloader",1089 "id": 214577,1090 "title": "DistNetworkError when using multiprocessing_context parameter in pytorch dataloader",1091 "slug": "distnetworkerror-when-using-multiprocessing-context-parameter-in-pytorch-dataloader",1092 "posts_count": 3,1093 "reply_count": 1,1094 "highest_post_number": 3,1095 "image_url": null,1096 "created_at": "2024-12-23T17:44:12.037Z",1097 "last_posted_at": "2025-01-08T14:38:53.740Z",1098 "bumped": true,1099 "bumped_at": "2025-01-08T14:38:53.740Z",1100 "archetype": "regular",1101 "unseen": false,1102 "pinned": false,1103 "unpinned": null,1104 "visible": true,1105 "closed": false,1106 "archived": false,1107 "bookmarked": null,1108 "liked": null,1109 "tags_descriptions": {},1110 "like_count": 0,1111 "views": 243,1112 "category_id": 12,1113 "featured_link": null,1114 "has_accepted_answer": true,1115 "posters": [1116 {1117 "extras": "latest",1118 "description": "Original Poster, Most Recent Poster, Accepted Answer",1119 "user": {1120 "id": 81406,1121 "username": "forestbat",1122 "name": "forestbat",1123 "avatar_template": "/user_avatar/discuss.pytorch.org/forestbat/{size}/74440_2.png",1124 "trust_level": 11125 }1126 },1127 {1128 "extras": null,1129 "description": "Frequent Poster",1130 "user": {1131 "id": 39542,1132 "username": "H-Huang",1133 "name": "Howard Huang",1134 "avatar_template": "/user_avatar/discuss.pytorch.org/h-huang/{size}/35598_2.png",1135 "trust_level": 21136 }1137 }1138 ]1139 },1140 {1141 "fancy_title": "How can I run 5 processes per GPU for three GPUs using DDP?",1142 "id": 215534,1143 "title": "How can I run 5 processes per GPU for three GPUs using DDP?",1144 "slug": "how-can-i-run-5-processes-per-gpu-for-three-gpus-using-ddp",1145 "posts_count": 4,1146 "reply_count": 2,1147 "highest_post_number": 4,1148 "image_url": null,1149 "created_at": "2025-01-18T02:26:01.681Z",1150 "last_posted_at": "2025-01-23T01:38:20.828Z",1151 "bumped": true,1152 "bumped_at": "2025-01-23T01:38:20.828Z",1153 "archetype": "regular",1154 "unseen": false,1155 "pinned": false,1156 "unpinned": null,1157 "visible": true,1158 "closed": false,1159 "archived": false,1160 "bookmarked": null,1161 "liked": null,1162 "tags_descriptions": {},1163 "like_count": 2,1164 "views": 89,1165 "category_id": 12,1166 "featured_link": null,1167 "has_accepted_answer": true,1168 "posters": [1169 {1170 "extras": "latest",1171 "description": "Original Poster, Most Recent Poster",1172 "user": {1173 "id": 70325,1174 "username": "yhl3051",1175 "name": "",1176 "avatar_template": "/user_avatar/discuss.pytorch.org/yhl3051/{size}/75168_2.png",1177 "trust_level": 11178 }1179 },1180 {1181 "extras": null,1182 "description": "Frequent Poster, Accepted Answer",1183 "user": {1184 "id": 3534,1185 "username": "ptrblck",1186 "name": "",1187 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1188 "admin": true,1189 "moderator": true,1190 "trust_level": 21191 }1192 }1193 ]1194 },1195 {1196 "fancy_title": "FSDP2 evaluate during training",1197 "id": 215240,1198 "title": "FSDP2 evaluate during training",1199 "slug": "fsdp2-evaluate-during-training",1200 "posts_count": 2,