Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 409754,7 "name": "YX",8 "username": "yxchng1",9 "avatar_template": "/letter_avatar_proxy/v4/letter/y/b9e5f3/{size}.png",10 "created_at": "2023-07-14T01:57:08.868Z",11 "cooked": "<p>What does <code>torch/distributed/fsdp/flat_param.py\", line 1999, in _check_storage_allocated p_assert(storage_size > 0, \"Expects storage to be allocated\")</code> assertion do?</p>\n<p>What can be the possible causes for storage size being 0?</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 4,15 "updated_at": "2023-07-14T01:57:08.868Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 373,20 "reads": 11,21 "readers_count": 10,22 "score": 1867.2,23 "yours": false,24 "topic_id": 184184,25 "topic_slug": "what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do",26 "display_username": "YX",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 60644,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do/184184/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 429183,64 "name": "Sujay S Kumar",65 "username": "sujayskumar",66 "avatar_template": "/user_avatar/discuss.pytorch.org/sujayskumar/{size}/66722_2.png",67 "created_at": "2024-01-05T00:13:37.645Z",68 "cooked": "<p>Getting the same error. Any ideas?</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 4,72 "updated_at": "2024-01-05T00:13:37.645Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 0,77 "reads": 6,78 "readers_count": 5,79 "score": 6.2,80 "yours": false,81 "topic_id": 184184,82 "topic_slug": "what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do",83 "display_username": "Sujay S Kumar",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": null,99 "bookmarked": false,100 "actions_summary": [],101 "moderator": false,102 "admin": false,103 "staff": false,104 "user_id": 72242,105 "hidden": false,106 "trust_level": 1,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do/184184/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 429187,120 "name": "",121 "username": "ptrblck",122 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",123 "created_at": "2024-01-05T02:01:36.039Z",124 "cooked": "<p>Could you post a minimal and executable code snippet reproducing the error, please?</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 4,128 "updated_at": "2024-01-05T02:01:36.039Z",129 "reply_count": 1,130 "reply_to_post_number": 2,131 "quote_count": 0,132 "incoming_link_count": 2,133 "reads": 6,134 "readers_count": 5,135 "score": 16.2,136 "yours": false,137 "topic_id": 184184,138 "topic_slug": "what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do",139 "display_username": "",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 1,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": "",155 "reply_to_user": {156 "id": 72242,157 "username": "sujayskumar",158 "name": "Sujay S Kumar",159 "avatar_template": "/user_avatar/discuss.pytorch.org/sujayskumar/{size}/66722_2.png"160 },161 "bookmarked": false,162 "actions_summary": [],163 "moderator": true,164 "admin": true,165 "staff": true,166 "user_id": 3534,167 "hidden": false,168 "trust_level": 2,169 "deleted_at": null,170 "user_deleted": false,171 "edit_reason": null,172 "can_view_edit_history": true,173 "wiki": false,174 "post_url": "/t/what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do/184184/3",175 "can_accept_answer": false,176 "can_unaccept_answer": false,177 "accepted_answer": false,178 "topic_accepted_answer": null179 },180 {181 "id": 445353,182 "name": "Shikhar Srivastava",183 "username": "shikhar-srivastava",184 "avatar_template": "/user_avatar/discuss.pytorch.org/shikhar-srivastava/{size}/42126_2.png",185 "created_at": "2024-06-09T17:23:15.372Z",186 "cooked": "<p>Hard to produce a minimal executable for this. Faced the same error.<br>\nRemoved the assert, and FSDP training seems to execute just fine.</p>\n<p>Would be helpful to understand what scenarios this assert solves for?</p>",187 "post_number": 4,188 "post_type": 1,189 "posts_count": 4,190 "updated_at": "2024-06-09T17:23:15.372Z",191 "reply_count": 0,192 "reply_to_post_number": 3,193 "quote_count": 0,194 "incoming_link_count": 2,195 "reads": 2,196 "readers_count": 1,197 "score": 10.4,198 "yours": false,199 "topic_id": 184184,200 "topic_slug": "what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do",201 "display_username": "Shikhar Srivastava",202 "primary_group_name": null,203 "flair_name": null,204 "flair_url": null,205 "flair_bg_color": null,206 "flair_color": null,207 "flair_group_id": null,208 "badges_granted": [],209 "version": 1,210 "can_edit": false,211 "can_delete": false,212 "can_recover": false,213 "can_see_hidden_post": false,214 "can_wiki": false,215 "read": true,216 "user_title": null,217 "reply_to_user": {218 "id": 3534,219 "username": "ptrblck",220 "name": "",221 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"222 },223 "bookmarked": false,224 "actions_summary": [],225 "moderator": false,226 "admin": false,227 "staff": false,228 "user_id": 48940,229 "hidden": false,230 "trust_level": 1,231 "deleted_at": null,232 "user_deleted": false,233 "edit_reason": null,234 "can_view_edit_history": true,235 "wiki": false,236 "post_url": "/t/what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do/184184/4",237 "can_accept_answer": false,238 "can_unaccept_answer": false,239 "accepted_answer": false,240 "topic_accepted_answer": null241 }242 ],243 "stream": [244 409754,245 429183,246 429187,247 445353248 ]249 },250 "timeline_lookup": [251 [252 1,253 835254 ],255 [256 2,257 660258 ],259 [260 4,261 503262 ]263 ],264 "suggested_topics": [265 {266 "fancy_title": "Int8 is working slower than float32 for matrix multiplication",267 "id": 214024,268 "title": "Int8 is working slower than float32 for matrix multiplication",269 "slug": "int8-is-working-slower-than-float32-for-matrix-multiplication",270 "posts_count": 1,271 "reply_count": 0,272 "highest_post_number": 1,273 "image_url": null,274 "created_at": "2024-12-10T04:39:13.767Z",275 "last_posted_at": "2024-12-10T04:39:13.816Z",276 "bumped": true,277 "bumped_at": "2024-12-10T07:17:55.129Z",278 "archetype": "regular",279 "unseen": false,280 "pinned": false,281 "unpinned": null,282 "visible": true,283 "closed": false,284 "archived": false,285 "bookmarked": null,286 "liked": null,287 "tags_descriptions": {},288 "like_count": 0,289 "views": 191,290 "category_id": 1,291 "featured_link": null,292 "has_accepted_answer": false,293 "posters": [294 {295 "extras": "latest single",296 "description": "Original Poster, Most Recent Poster",297 "user": {298 "id": 81419,299 "username": "brijesh_bhalodiya",300 "name": "Brijesh Bhalodiya ",301 "avatar_template": "/user_avatar/discuss.pytorch.org/brijesh_bhalodiya/{size}/74321_2.png",302 "trust_level": 1303 }304 }305 ]306 },307 {308 "fancy_title": "Understanding the traceback of a compiler error with cache size limit",309 "id": 215669,310 "title": "Understanding the traceback of a compiler error with cache size limit",311 "slug": "understanding-the-traceback-of-a-compiler-error-with-cache-size-limit",312 "posts_count": 1,313 "reply_count": 0,314 "highest_post_number": 1,315 "image_url": null,316 "created_at": "2025-01-21T10:43:52.773Z",317 "last_posted_at": "2025-01-21T10:43:52.812Z",318 "bumped": true,319 "bumped_at": "2025-01-21T10:43:52.812Z",320 "archetype": "regular",321 "unseen": false,322 "pinned": false,323 "unpinned": null,324 "visible": true,325 "closed": false,326 "archived": false,327 "bookmarked": null,328 "liked": null,329 "tags_descriptions": {},330 "like_count": 0,331 "views": 53,332 "category_id": 1,333 "featured_link": null,334 "has_accepted_answer": false,335 "posters": [336 {337 "extras": "latest single",338 "description": "Original Poster, Most Recent Poster",339 "user": {340 "id": 82217,341 "username": "Johannes_Vogt",342 "name": "Johannes Vogt",343 "avatar_template": "/user_avatar/discuss.pytorch.org/johannes_vogt/{size}/75220_2.png",344 "trust_level": 1345 }346 }347 ]348 },349 {350 "fancy_title": "Restarting a distributedDataParallel",351 "id": 215799,352 "title": "Restarting a distributedDataParallel",353 "slug": "restarting-a-distributeddataparallel",354 "posts_count": 2,355 "reply_count": 0,356 "highest_post_number": 2,357 "image_url": null,358 "created_at": "2025-01-24T01:56:22.157Z",359 "last_posted_at": "2025-01-24T02:30:45.502Z",360 "bumped": true,361 "bumped_at": "2025-01-24T02:30:45.502Z",362 "archetype": "regular",363 "unseen": false,364 "pinned": false,365 "unpinned": null,366 "visible": true,367 "closed": false,368 "archived": false,369 "bookmarked": null,370 "liked": null,371 "tags_descriptions": {},372 "like_count": 0,373 "views": 30,374 "category_id": 1,375 "featured_link": null,376 "has_accepted_answer": true,377 "posters": [378 {379 "extras": "latest single",380 "description": "Original Poster, Most Recent Poster, Accepted Answer",381 "user": {382 "id": 6296,383 "username": "pytorcher",384 "name": "",385 "avatar_template": "/letter_avatar_proxy/v4/letter/p/7ea924/{size}.png",386 "trust_level": 1387 }388 }389 ]390 },391 {392 "fancy_title": "Project Sharing: Easy Imitation Learning using Pytorch with myCobot",393 "id": 216621,394 "title": "Project Sharing: Easy Imitation Learning using Pytorch with myCobot",395 "slug": "project-sharing-easy-imitation-learning-using-pytorch-with-mycobot",396 "posts_count": 1,397 "reply_count": 0,398 "highest_post_number": 1,399 "image_url": null,400 "created_at": "2025-02-13T09:28:05.397Z",401 "last_posted_at": "2025-02-13T09:28:05.445Z",402 "bumped": true,403 "bumped_at": "2025-02-13T09:28:05.445Z",404 "archetype": "regular",405 "unseen": false,406 "pinned": false,407 "unpinned": null,408 "visible": true,409 "closed": false,410 "archived": false,411 "bookmarked": null,412 "liked": null,413 "tags_descriptions": {},414 "like_count": 1,415 "views": 350,416 "category_id": 1,417 "featured_link": null,418 "has_accepted_answer": false,419 "posters": [420 {421 "extras": "latest single",422 "description": "Original Poster, Most Recent Poster",423 "user": {424 "id": 82641,425 "username": "ElephantRobotics123",426 "name": "Mavis Huang",427 "avatar_template": "/letter_avatar_proxy/v4/letter/e/7feea3/{size}.png",428 "trust_level": 0429 }430 }431 ]432 },433 {434 "fancy_title": "How to Selectively Clear Gradients Accumulated from Previous Time Steps in RNN Backpropagation?",435 "id": 216888,436 "title": "How to Selectively Clear Gradients Accumulated from Previous Time Steps in RNN Backpropagation?",437 "slug": "how-to-selectively-clear-gradients-accumulated-from-previous-time-steps-in-rnn-backpropagation",438 "posts_count": 1,439 "reply_count": 0,440 "highest_post_number": 1,441 "image_url": null,442 "created_at": "2025-02-19T12:13:58.278Z",443 "last_posted_at": "2025-02-19T12:13:58.320Z",444 "bumped": true,445 "bumped_at": "2025-02-19T12:44:07.484Z",446 "archetype": "regular",447 "unseen": false,448 "pinned": false,449 "unpinned": null,450 "visible": true,451 "closed": false,452 "archived": false,453 "bookmarked": null,454 "liked": null,455 "tags_descriptions": {},456 "like_count": 0,457 "views": 56,458 "category_id": 1,459 "featured_link": null,460 "has_accepted_answer": false,461 "posters": [462 {463 "extras": "latest single",464 "description": "Original Poster, Most Recent Poster",465 "user": {466 "id": 68895,467 "username": "DawidL",468 "name": "",469 "avatar_template": "/user_avatar/discuss.pytorch.org/dawidl/{size}/62949_2.png",470 "trust_level": 1471 }472 }473 ]474 }475 ],476 "tags_descriptions": {},477 "fancy_title": "What does `torch/distributed/fsdp/flat_param.py”, line 1999, in _check_storage_allocated p_assert(storage_size > 0, “Expects storage to be allocated”)` assertion do?",478 "id": 184184,479 "title": "What does `torch/distributed/fsdp/flat_param.py\", line 1999, in _check_storage_allocated p_assert(storage_size > 0, \"Expects storage to be allocated\")` assertion do?",480 "posts_count": 4,481 "created_at": "2023-07-14T01:57:08.748Z",482 "views": 550,483 "reply_count": 2,484 "like_count": 0,485 "last_posted_at": "2024-06-09T17:23:15.372Z",486 "visible": true,487 "closed": false,488 "archived": false,489 "has_summary": false,490 "archetype": "regular",491 "slug": "what-does-torch-distributed-fsdp-flat-param-py-line-1999-in-check-storage-allocated-p-assert-storage-size-0-expects-storage-to-be-allocated-assertion-do",492 "category_id": 1,493 "word_count": 85,494 "deleted_at": null,495 "user_id": 60644,496 "featured_link": null,497 "pinned_globally": false,498 "pinned_at": null,499 "pinned_until": null,500 "image_url": null,501 "slow_mode_seconds": 0,502 "draft": null,503 "draft_key": "topic_184184",504 "draft_sequence": null,505 "unpinned": null,506 "pinned": false,507 "current_post_number": 1,508 "highest_post_number": 4,509 "deleted_by": null,510 "actions_summary": [511 {512 "id": 4,513 "count": 0,514 "hidden": false,515 "can_act": false516 },517 {518 "id": 8,519 "count": 0,520 "hidden": false,521 "can_act": false522 },523 {524 "id": 10,525 "count": 0,526 "hidden": false,527 "can_act": false528 },529 {530 "id": 7,531 "count": 0,532 "hidden": false,533 "can_act": false534 }535 ],536 "chunk_size": 20,537 "bookmarked": false,538 "topic_timer": null,539 "message_bus_last_id": 0,540 "participant_count": 4,541 "show_read_indicator": false,542 "thumbnails": null,543 "slow_mode_enabled_until": null,544 "can_vote": false,545 "vote_count": 0,546 "user_voted": false,547 "discourse_zendesk_plugin_zendesk_id": null,548 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",549 "details": {550 "can_edit": false,551 "notification_level": 1,552 "participants": [553 {554 "id": 3534,555 "username": "ptrblck",556 "name": "",557 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",558 "post_count": 1,559 "primary_group_name": null,560 "flair_name": null,561 "flair_url": null,562 "flair_color": null,563 "flair_bg_color": null,564 "flair_group_id": null,565 "admin": true,566 "moderator": true,567 "trust_level": 2568 },569 {570 "id": 48940,571 "username": "shikhar-srivastava",572 "name": "Shikhar Srivastava",573 "avatar_template": "/user_avatar/discuss.pytorch.org/shikhar-srivastava/{size}/42126_2.png",574 "post_count": 1,575 "primary_group_name": null,576 "flair_name": null,577 "flair_url": null,578 "flair_color": null,579 "flair_bg_color": null,580 "flair_group_id": null,581 "trust_level": 1582 },583 {584 "id": 60644,585 "username": "yxchng1",586 "name": "YX",587 "avatar_template": "/letter_avatar_proxy/v4/letter/y/b9e5f3/{size}.png",588 "post_count": 1,589 "primary_group_name": null,590 "flair_name": null,591 "flair_url": null,592 "flair_color": null,593 "flair_bg_color": null,594 "flair_group_id": null,595 "trust_level": 1596 },597 {598 "id": 72242,599 "username": "sujayskumar",600 "name": "Sujay S Kumar",601 "avatar_template": "/user_avatar/discuss.pytorch.org/sujayskumar/{size}/66722_2.png",602 "post_count": 1,603 "primary_group_name": null,604 "flair_name": null,605 "flair_url": null,606 "flair_color": null,607 "flair_bg_color": null,608 "flair_group_id": null,609 "trust_level": 1610 }611 ],612 "created_by": {613 "id": 60644,614 "username": "yxchng1",615 "name": "YX",616 "avatar_template": "/letter_avatar_proxy/v4/letter/y/b9e5f3/{size}.png"617 },618 "last_poster": {619 "id": 48940,620 "username": "shikhar-srivastava",621 "name": "Shikhar Srivastava",622 "avatar_template": "/user_avatar/discuss.pytorch.org/shikhar-srivastava/{size}/42126_2.png"623 }624 },625 "bookmarks": []626 },627 {628 "post_stream": {629 "posts": [630 {631 "id": 445316,632 "name": "Mark",633 "username": "BKram",634 "avatar_template": "/letter_avatar_proxy/v4/letter/b/cab0a1/{size}.png",635 "created_at": "2024-06-08T19:43:38.544Z",636 "cooked": "<p>Hi All,</p>\n<p>I’m new to this forum, and also quite new to Pytorch.</p>\n<p>I’m running into an issue that Data loader seems to be quite slow and i’m not sure what the bottle neck is. running on a ryzen 5700, 32 GB mem and a 4700 Super.</p>\n<pre><code class=\"lang-auto\">device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\ntransform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\n# Load CIFAR-10 dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n\ndef profile_data_loading(train_loader):\n start_time = time.time() # Start timing\n \n for batch_idx, (data, target) in enumerate(train_loader):\n if batch_idx == 0:\n load_time = time.time() - start_time # End timing\n print(f\"Loading time: {load_time} seconds\")\n pass\n \n loop_time = time.time() - start_time - load_time\n print(f\"Loop time: {loop_time} seconds\")\n\ntrain_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=2, pin_memory=True)\n\n</code></pre>\n<p>On my pc I get:<br>\ncuda<br>\nFiles already downloaded and verified<br>\nLoading time: 5.744622468948364 seconds<br>\nLoop time: 4.3751795291900635 seconds</p>\n<p>On colab I get<br>\ncuda<br>\nFiles already downloaded and verified<br>\nLoading time: 0.11000442504882812 seconds<br>\nLoop time: 13.87480902671814 seconds</p>\n<p>What makes the loading time on my pc so slow, or is this to be expected with my setup?</p>\n<p>Thanks,</p>\n<blockquote>\n<p>Blockquote</p>\n</blockquote>",637 "post_number": 1,638 "post_type": 1,639 "posts_count": 10,640 "updated_at": "2024-06-08T19:43:38.544Z",641 "reply_count": 0,642 "reply_to_post_number": null,643 "quote_count": 0,644 "incoming_link_count": 841,645 "reads": 16,646 "readers_count": 15,647 "score": 4168.2,648 "yours": false,649 "topic_id": 204297,650 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",651 "display_username": "Mark",652 "primary_group_name": null,653 "flair_name": null,654 "flair_url": null,655 "flair_bg_color": null,656 "flair_color": null,657 "flair_group_id": null,658 "badges_granted": [],659 "version": 1,660 "can_edit": false,661 "can_delete": false,662 "can_recover": false,663 "can_see_hidden_post": false,664 "can_wiki": false,665 "read": true,666 "user_title": null,667 "bookmarked": false,668 "actions_summary": [],669 "moderator": false,670 "admin": false,671 "staff": false,672 "user_id": 76488,673 "hidden": false,674 "trust_level": 1,675 "deleted_at": null,676 "user_deleted": false,677 "edit_reason": null,678 "can_view_edit_history": true,679 "wiki": false,680 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/1",681 "can_accept_answer": false,682 "can_unaccept_answer": false,683 "accepted_answer": false,684 "topic_accepted_answer": null,685 "can_vote": false686 },687 {688 "id": 445330,689 "name": "Mark Saroufim",690 "username": "marksaroufim",691 "avatar_template": "/user_avatar/discuss.pytorch.org/marksaroufim/{size}/36747_2.png",692 "created_at": "2024-06-09T06:00:34.724Z",693 "cooked": "<p>First thing I’d try is varying the <code>num_workers</code>, try setting it to 0 and slowly increasing it and see if the pattern changes</p>\n<p>Also just confirming since this is a local setup, do you have an SSD or HDD?</p>",694 "post_number": 2,695 "post_type": 1,696 "posts_count": 10,697 "updated_at": "2024-06-09T06:00:34.724Z",698 "reply_count": 1,699 "reply_to_post_number": null,700 "quote_count": 0,701 "incoming_link_count": 0,702 "reads": 12,703 "readers_count": 11,704 "score": 7.4,705 "yours": false,706 "topic_id": 204297,707 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",708 "display_username": "Mark Saroufim",709 "primary_group_name": null,710 "flair_name": null,711 "flair_url": null,712 "flair_bg_color": null,713 "flair_color": null,714 "flair_group_id": null,715 "badges_granted": [],716 "version": 1,717 "can_edit": false,718 "can_delete": false,719 "can_recover": false,720 "can_see_hidden_post": false,721 "can_wiki": false,722 "read": true,723 "user_title": "Regular",724 "title_is_group": false,725 "bookmarked": false,726 "actions_summary": [],727 "moderator": false,728 "admin": false,729 "staff": false,730 "user_id": 43941,731 "hidden": false,732 "trust_level": 3,733 "deleted_at": null,734 "user_deleted": false,735 "edit_reason": null,736 "can_view_edit_history": true,737 "wiki": false,738 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/2",739 "can_accept_answer": false,740 "can_unaccept_answer": false,741 "accepted_answer": false,742 "topic_accepted_answer": null743 },744 {745 "id": 445333,746 "name": "Mark",747 "username": "BKram",748 "avatar_template": "/letter_avatar_proxy/v4/letter/b/cab0a1/{size}.png",749 "created_at": "2024-06-09T07:25:22.118Z",750 "cooked": "<p>Thanks for you reply,</p>\n<p>I’ve played around with that a bit, I get a huge increase in performance when I set num_workers to 0, reduces the Loading time to about 0.013!<br>\nincreasing it to 1, I get 2.8 sec. Increasing further to 22 seconds at 8. Does this makes sense?</p>\n<p>Increasing the num_workers lowers the loop time from 7.5 sec at 0 workers to 2 sec at 8 workers.</p>\n<p>I’m using an SSD</p>\n<p>Thanks,</p>",751 "post_number": 3,752 "post_type": 1,753 "posts_count": 10,754 "updated_at": "2024-06-09T07:25:22.118Z",755 "reply_count": 0,756 "reply_to_post_number": 2,757 "quote_count": 0,758 "incoming_link_count": 2,759 "reads": 12,760 "readers_count": 11,761 "score": 27.4,762 "yours": false,763 "topic_id": 204297,764 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",765 "display_username": "Mark",766 "primary_group_name": null,767 "flair_name": null,768 "flair_url": null,769 "flair_bg_color": null,770 "flair_color": null,771 "flair_group_id": null,772 "badges_granted": [],773 "version": 1,774 "can_edit": false,775 "can_delete": false,776 "can_recover": false,777 "can_see_hidden_post": false,778 "can_wiki": false,779 "read": true,780 "user_title": null,781 "reply_to_user": {782 "id": 43941,783 "username": "marksaroufim",784 "name": "Mark Saroufim",785 "avatar_template": "/user_avatar/discuss.pytorch.org/marksaroufim/{size}/36747_2.png"786 },787 "bookmarked": false,788 "actions_summary": [789 {790 "id": 2,791 "count": 1792 }793 ],794 "moderator": false,795 "admin": false,796 "staff": false,797 "user_id": 76488,798 "hidden": false,799 "trust_level": 1,800 "deleted_at": null,801 "user_deleted": false,802 "edit_reason": null,803 "can_view_edit_history": true,804 "wiki": false,805 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/3",806 "can_accept_answer": false,807 "can_unaccept_answer": false,808 "accepted_answer": false,809 "topic_accepted_answer": null810 },811 {812 "id": 445334,813 "name": "",814 "username": "mMagmer",815 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png",816 "created_at": "2024-06-09T10:32:22.649Z",817 "cooked": "<p>these are the results on my machine : torch 2.1 ubuntu 18.04 ryzen 3600 and on a HDD drive</p>\n<pre><code class=\"lang-auto\"># num_workers 0\nprofile_data_loading(train_loader)\n#>> Loading time: 0.5861883163452148 seconds\n#>> Loop time: 6.6978724002838135 seconds\n# num_workers 2\nprofile_data_loading(train_loader)\n#>> Loading time: 0.6597647666931152 seconds\n#>> Loop time: 3.702897071838379 seconds\n# num_workers 4\nprofile_data_loading(train_loader)\n#>> Loading time: 0.6669812202453613 seconds\n#>> Loop time: 2.085876941680908 seconds\n# num_workers 8\nprofile_data_loading(train_loader)\n#>> Loading time: 0.7236223220825195 seconds\n#>> Loop time: 1.2904746532440186 seconds\n</code></pre>\n<p>also the results for next epochs are usually much faster. i think dataloader or OS is Caching some information about the location of the file on the storage.<br>\nare you using a Windows machine??</p>",818 "post_number": 4,819 "post_type": 1,820 "posts_count": 10,821 "updated_at": "2024-06-09T10:32:22.649Z",822 "reply_count": 0,823 "reply_to_post_number": null,824 "quote_count": 0,825 "incoming_link_count": 3,826 "reads": 12,827 "readers_count": 11,828 "score": 17.4,829 "yours": false,830 "topic_id": 204297,831 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",832 "display_username": "",833 "primary_group_name": null,834 "flair_name": null,835 "flair_url": null,836 "flair_bg_color": null,837 "flair_color": null,838 "flair_group_id": null,839 "badges_granted": [],840 "version": 1,841 "can_edit": false,842 "can_delete": false,843 "can_recover": false,844 "can_see_hidden_post": false,845 "can_wiki": false,846 "read": true,847 "user_title": "",848 "bookmarked": false,849 "actions_summary": [],850 "moderator": false,851 "admin": false,852 "staff": false,853 "user_id": 50063,854 "hidden": false,855 "trust_level": 2,856 "deleted_at": null,857 "user_deleted": false,858 "edit_reason": null,859 "can_view_edit_history": true,860 "wiki": false,861 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/4",862 "can_accept_answer": false,863 "can_unaccept_answer": false,864 "accepted_answer": false,865 "topic_accepted_answer": null866 },867 {868 "id": 445336,869 "name": "Mark",870 "username": "BKram",871 "avatar_template": "/letter_avatar_proxy/v4/letter/b/cab0a1/{size}.png",872 "created_at": "2024-06-09T10:53:02.278Z",873 "cooked": "<p>I’m using a windows 11 machine with python 3.10.14 and Pytorch 2.2.2.<br>\nYour results look much more consistent with or without the data loader than mine.</p>\n<p>Looking at my system resources I see very limited activity on PC, for 8 workers:<br>\nSSD: no notable percentage difference in use (1~2% of max)<br>\nCPU: 10%<br>\nMem: 3GB increase (14.7 to 17.7 GB) in use.</p>\n<p>What could be limiting my performance compared to yours?</p>",874 "post_number": 5,875 "post_type": 1,876 "posts_count": 10,877 "updated_at": "2024-06-09T10:53:02.278Z",878 "reply_count": 0,879 "reply_to_post_number": null,880 "quote_count": 0,881 "incoming_link_count": 10,882 "reads": 9,883 "readers_count": 8,884 "score": 51.8,885 "yours": false,886 "topic_id": 204297,887 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",888 "display_username": "Mark",889 "primary_group_name": null,890 "flair_name": null,891 "flair_url": null,892 "flair_bg_color": null,893 "flair_color": null,894 "flair_group_id": null,895 "badges_granted": [],896 "version": 1,897 "can_edit": false,898 "can_delete": false,899 "can_recover": false,900 "can_see_hidden_post": false,901 "can_wiki": false,902 "read": true,903 "user_title": null,904 "bookmarked": false,905 "actions_summary": [],906 "moderator": false,907 "admin": false,908 "staff": false,909 "user_id": 76488,910 "hidden": false,911 "trust_level": 1,912 "deleted_at": null,913 "user_deleted": false,914 "edit_reason": null,915 "can_view_edit_history": true,916 "wiki": false,917 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/5",918 "can_accept_answer": false,919 "can_unaccept_answer": false,920 "accepted_answer": false,921 "topic_accepted_answer": null922 },923 {924 "id": 445338,925 "name": "",926 "username": "mMagmer",927 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png",928 "created_at": "2024-06-09T11:19:00.858Z",929 "cooked": "<p>take look at this <a href=\"https://pytorch.org/docs/stable/data.html#platform-specific-behaviors\" rel=\"noopener nofollow ugc\">https://pytorch.org/docs/stable/data.html#platform-specific-behaviors</a></p>\n<blockquote>\n<p>This separate serialization means that you should take two steps to ensure you are compatible with Windows while using multi-process data loading:</p>\n<ul>\n<li>Wrap most of you main script’s code within <code>if __name__ == '__main__':</code> block, to make sure it doesn’t run again (most likely generating error) when each worker process is launched. You can place your dataset and <a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader\" rel=\"noopener nofollow ugc\"><code>DataLoader</code></a> instance creation logic here, as it doesn’t need to be re-executed in workers.</li>\n<li>Make sure that any custom <code>collate_fn</code>, <code>worker_init_fn</code> or <a href=\"https://pytorch.org/docs/stable/utils.html#module-torch.utils.data.dataset\" rel=\"noopener nofollow ugc\"><code>dataset</code></a> code is declared as top level definitions, outside of the <code>__main__</code> check. This ensures that they are available in worker processes. (this is needed since functions are pickled as references only, not <code>bytecode</code>.)</li>\n</ul>\n</blockquote>\n<p>i think for this to work you can’t use jupyter notebook.<br>\nwrite .py script.</p>",930 "post_number": 6,931 "post_type": 1,932 "posts_count": 10,933 "updated_at": "2024-06-09T11:19:00.858Z",934 "reply_count": 2,935 "reply_to_post_number": null,936 "quote_count": 0,937 "incoming_link_count": 8,938 "reads": 10,939 "readers_count": 9,940 "score": 52.0,941 "yours": false,942 "topic_id": 204297,943 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",944 "display_username": "",945 "primary_group_name": null,946 "flair_name": null,947 "flair_url": null,948 "flair_bg_color": null,949 "flair_color": null,950 "flair_group_id": null,951 "badges_granted": [],952 "version": 1,953 "can_edit": false,954 "can_delete": false,955 "can_recover": false,956 "can_see_hidden_post": false,957 "can_wiki": false,958 "link_counts": [959 {960 "url": "https://pytorch.org/docs/stable/data.html#platform-specific-behaviors",961 "internal": false,962 "reflection": false,963 "title": "torch.utils.data — PyTorch 2.3 documentation",964 "clicks": 20965 },966 {967 "url": "https://pytorch.org/docs/stable/utils.html#module-torch.utils.data.dataset",968 "internal": false,969 "reflection": false,970 "title": "torch.utils — PyTorch 2.3 documentation",971 "clicks": 1972 },973 {974 "url": "https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader",975 "internal": false,976 "reflection": false,977 "title": "torch.utils.data — PyTorch 2.3 documentation",978 "clicks": 0979 }980 ],981 "read": true,982 "user_title": "",983 "bookmarked": false,984 "actions_summary": [],985 "moderator": false,986 "admin": false,987 "staff": false,988 "user_id": 50063,989 "hidden": false,990 "trust_level": 2,991 "deleted_at": null,992 "user_deleted": false,993 "edit_reason": null,994 "can_view_edit_history": true,995 "wiki": false,996 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/6",997 "can_accept_answer": false,998 "can_unaccept_answer": false,999 "accepted_answer": false,1000 "topic_accepted_answer": null1001 },1002 {1003 "id": 445341,1004 "name": "Mark",1005 "username": "BKram",1006 "avatar_template": "/letter_avatar_proxy/v4/letter/b/cab0a1/{size}.png",1007 "created_at": "2024-06-09T13:08:11.835Z",1008 "cooked": "<p>Thanks for your input, chagend the script accordingly:</p>\n<pre><code class=\"lang-auto\">import time\nimport torch\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms\n\ndef profile_data_loading(train_loader):\n start_time = time.time() # Start timing\n\n for batch_idx, (data, target) in enumerate(train_loader):\n if batch_idx == 0:\n load_time = time.time() - start_time # End timing\n print(f\"Loading time: {load_time} seconds\")\n pass\n\n loop_time = time.time() - start_time - load_time\n print(f\"Loop time: {loop_time} seconds\")\n\ndef main():\n device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n print(device)\n\n transform = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n ])\n\n # Load CIFAR-10 dataset\n train_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=8, pin_memory=True)\n\n profile_data_loading(train_loader)\n\nif __name__ == \"__main__\":\n main()\n</code></pre>\n<p>However, no impact on the results unfortunately.<br>\nStill 22 seconds at 8 workers.</p>\n<p>Any other suggestions?</p>",1009 "post_number": 7,1010 "post_type": 1,1011 "posts_count": 10,1012 "updated_at": "2024-06-09T13:19:02.552Z",1013 "reply_count": 1,1014 "reply_to_post_number": 6,1015 "quote_count": 0,1016 "incoming_link_count": 6,1017 "reads": 10,1018 "readers_count": 9,1019 "score": 37.0,1020 "yours": false,1021 "topic_id": 204297,1022 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",1023 "display_username": "Mark",1024 "primary_group_name": null,1025 "flair_name": null,1026 "flair_url": null,1027 "flair_bg_color": null,1028 "flair_color": null,1029 "flair_group_id": null,1030 "badges_granted": [],1031 "version": 2,1032 "can_edit": false,1033 "can_delete": false,1034 "can_recover": false,1035 "can_see_hidden_post": false,1036 "can_wiki": false,1037 "read": true,1038 "user_title": null,1039 "reply_to_user": {1040 "id": 50063,1041 "username": "mMagmer",1042 "name": "",1043 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png"1044 },1045 "bookmarked": false,1046 "actions_summary": [],1047 "moderator": false,1048 "admin": false,1049 "staff": false,1050 "user_id": 76488,1051 "hidden": false,1052 "trust_level": 1,1053 "deleted_at": null,1054 "user_deleted": false,1055 "edit_reason": null,1056 "can_view_edit_history": true,1057 "wiki": false,1058 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/7",1059 "can_accept_answer": false,1060 "can_unaccept_answer": false,1061 "accepted_answer": false,1062 "topic_accepted_answer": null1063 },1064 {1065 "id": 445342,1066 "name": "",1067 "username": "mMagmer",1068 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png",1069 "created_at": "2024-06-09T13:19:26.655Z",1070 "cooked": "<aside class=\"quote no-group\" data-username=\"mMagmer\" data-post=\"6\" data-topic=\"204297\">\n<div class=\"title\">\n<div class=\"quote-controls\"></div>\n<img loading=\"lazy\" alt=\"\" width=\"24\" height=\"24\" src=\"https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/mmagmer/48/43283_2.png\" class=\"avatar\"> mMagmer:</div>\n<blockquote>\n<p>Make sure that any custom <code>collate_fn</code>, <code>worker_init_fn</code> or <a href=\"https://pytorch.org/docs/stable/utils.html#module-torch.utils.data.dataset\" rel=\"noopener nofollow ugc\"><code>dataset</code></a> code is declared as top level definitions, outside of the <code>__main__</code> check.</p>\n</blockquote>\n</aside>\n<p>maybe try to define dataset outside of main.<br>\nthis part:</p>\n<aside class=\"quote no-group\" data-username=\"BKram\" data-post=\"7\" data-topic=\"204297\">\n<div class=\"title\">\n<div class=\"quote-controls\"></div>\n<img loading=\"lazy\" alt=\"\" width=\"24\" height=\"24\" src=\"https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/b/cab0a1/48.png\" class=\"avatar\"> BKram:</div>\n<blockquote>\n<pre><code class=\"lang-auto\"> train_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n</code></pre>\n</blockquote>\n</aside>",1071 "post_number": 8,1072 "post_type": 1,1073 "posts_count": 10,1074 "updated_at": "2024-06-09T13:19:26.655Z",1075 "reply_count": 1,1076 "reply_to_post_number": 7,1077 "quote_count": 2,1078 "incoming_link_count": 8,1079 "reads": 10,1080 "readers_count": 9,1081 "score": 47.0,1082 "yours": false,1083 "topic_id": 204297,1084 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",1085 "display_username": "",1086 "primary_group_name": null,1087 "flair_name": null,1088 "flair_url": null,1089 "flair_bg_color": null,1090 "flair_color": null,1091 "flair_group_id": null,1092 "badges_granted": [],1093 "version": 1,1094 "can_edit": false,1095 "can_delete": false,1096 "can_recover": false,1097 "can_see_hidden_post": false,1098 "can_wiki": false,1099 "read": true,1100 "user_title": "",1101 "bookmarked": false,1102 "actions_summary": [],1103 "moderator": false,1104 "admin": false,1105 "staff": false,1106 "user_id": 50063,1107 "hidden": false,1108 "trust_level": 2,1109 "deleted_at": null,1110 "user_deleted": false,1111 "edit_reason": null,1112 "can_view_edit_history": true,1113 "wiki": false,1114 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/8",1115 "can_accept_answer": false,1116 "can_unaccept_answer": false,1117 "accepted_answer": false,1118 "topic_accepted_answer": null1119 },1120 {1121 "id": 445343,1122 "name": "",1123 "username": "mMagmer",1124 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png",1125 "created_at": "2024-06-09T13:23:28.310Z",1126 "cooked": "<p>i found this as well</p><aside class=\"quote\" data-post=\"5\" data-topic=\"97564\">\n <div class=\"title\">\n <div class=\"quote-controls\"></div>\n <img loading=\"lazy\" alt=\"\" width=\"24\" height=\"24\" src=\"https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/bramvanroy/48/28628_2.png\" class=\"avatar\">\n <a href=\"https://discuss.pytorch.org/t/errors-when-using-num-workers-0-in-dataloader/97564/5\">Errors when using num_workers>0 in DataLoader</a> \n </div>\n <blockquote>\n You can have a look here: <a href=\"https://docs.python.org/3.7/library/multiprocessing.html#multiprocessing-programming\" rel=\"noopener nofollow ugc\">https://docs.python.org/3.7/library/multiprocessing.html#multiprocessing-programming</a> \nNote that Windows can still cause issues here, be it pickling errors or slow performance. See <a href=\"https://github.com/pytorch/pytorch/issues/12831\" rel=\"noopener nofollow ugc\">https://github.com/pytorch/pytorch/issues/12831</a>. The safest approach is using 0 on Windows.\n </blockquote>\n</aside>\n",1127 "post_number": 9,1128 "post_type": 1,1129 "posts_count": 10,1130 "updated_at": "2024-06-09T13:23:28.310Z",1131 "reply_count": 0,1132 "reply_to_post_number": 8,1133 "quote_count": 0,1134 "incoming_link_count": 21,1135 "reads": 10,1136 "readers_count": 9,1137 "score": 107.0,1138 "yours": false,1139 "topic_id": 204297,1140 "topic_slug": "dataloader-slow-on-pc-fast-on-colab",1141 "display_username": "",1142 "primary_group_name": null,1143 "flair_name": null,1144 "flair_url": null,1145 "flair_bg_color": null,1146 "flair_color": null,1147 "flair_group_id": null,1148 "badges_granted": [],1149 "version": 1,1150 "can_edit": false,1151 "can_delete": false,1152 "can_recover": false,1153 "can_see_hidden_post": false,1154 "can_wiki": false,1155 "link_counts": [1156 {1157 "url": "https://discuss.pytorch.org/t/errors-when-using-num-workers-0-in-dataloader/97564/5",1158 "internal": true,1159 "reflection": false,1160 "title": "Errors when using num_workers>0 in DataLoader",1161 "clicks": 01162 }1163 ],1164 "read": true,1165 "user_title": "",1166 "reply_to_user": {1167 "id": 50063,1168 "username": "mMagmer",1169 "name": "",1170 "avatar_template": "/user_avatar/discuss.pytorch.org/mmagmer/{size}/43283_2.png"1171 },1172 "bookmarked": false,1173 "actions_summary": [],1174 "moderator": false,1175 "admin": false,1176 "staff": false,1177 "user_id": 50063,1178 "hidden": false,1179 "trust_level": 2,1180 "deleted_at": null,1181 "user_deleted": false,1182 "edit_reason": null,1183 "can_view_edit_history": true,1184 "wiki": false,1185 "post_url": "/t/dataloader-slow-on-pc-fast-on-colab/204297/9",1186 "can_accept_answer": false,1187 "can_unaccept_answer": false,1188 "accepted_answer": false,1189 "topic_accepted_answer": null1190 },1191 {1192 "id": 445348,1193 "name": "Mark",1194 "username": "BKram",1195 "avatar_template": "/letter_avatar_proxy/v4/letter/b/cab0a1/{size}.png",1196 "created_at": "2024-06-09T15:41:26.618Z",1197 "cooked": "<p>Tried defining the dataset outside <strong>main</strong>, but no effect unfortunately.<br>\nReading the link you’ve provided, seems to be a long running issue on Windows that was never resolved. Guess that sticking at 0 num_workers (or running on Linux) is the best way forward.</p>\n<p>Thanks for the assistance!</p>",1198 "post_number": 10,1199 "post_type": 1,1200 "posts_count": 10,