Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 97781,7 "name": "",8 "username": "Albert65",9 "avatar_template": "/user_avatar/discuss.pytorch.org/albert65/{size}/10175_2.png",10 "created_at": "2019-03-06T20:49:37.951Z",11 "cooked": "<p>Hi All, I would appreciate an example how to create a sparse Linear layer, which is similar to fully connected one with some links absent. It turns out the “torch.sparse” should be used, but I do not quite understand how to achieve that. I start from the dense tensor (image in my case), the next (hidden) layer shoud be a dense image of smaller size, and so on following the autoencoder layout. The only difference with the conventional pattern is that not every pair of entries of successive layers is connected (for performance reason).</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 6,15 "updated_at": "2019-03-06T20:49:37.951Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 6646,20 "reads": 151,21 "readers_count": 150,22 "score": 33240.2,23 "yours": false,24 "topic_id": 39150,25 "topic_slug": "sparse-linear-layer",26 "display_username": "",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [44 {45 "id": 2,46 "count": 247 }48 ],49 "moderator": false,50 "admin": false,51 "staff": false,52 "user_id": 16402,53 "hidden": false,54 "trust_level": 1,55 "deleted_at": null,56 "user_deleted": false,57 "edit_reason": null,58 "can_view_edit_history": true,59 "wiki": false,60 "post_url": "/t/sparse-linear-layer/39150/1",61 "can_accept_answer": false,62 "can_unaccept_answer": false,63 "accepted_answer": false,64 "topic_accepted_answer": null,65 "can_vote": false66 },67 {68 "id": 138601,69 "name": "Kale Ab Tessera",70 "username": "Kale-ab_Tessera",71 "avatar_template": "/user_avatar/discuss.pytorch.org/kale-ab_tessera/{size}/11407_2.png",72 "created_at": "2019-10-07T18:06:41.101Z",73 "cooked": "<p>Any news on this? I am also looking for sparse linear layers.</p>",74 "post_number": 2,75 "post_type": 1,76 "posts_count": 6,77 "updated_at": "2019-10-07T18:06:41.101Z",78 "reply_count": 0,79 "reply_to_post_number": null,80 "quote_count": 0,81 "incoming_link_count": 313,82 "reads": 139,83 "readers_count": 138,84 "score": 1592.8,85 "yours": false,86 "topic_id": 39150,87 "topic_slug": "sparse-linear-layer",88 "display_username": "Kale Ab Tessera",89 "primary_group_name": null,90 "flair_name": null,91 "flair_url": null,92 "flair_bg_color": null,93 "flair_color": null,94 "flair_group_id": null,95 "badges_granted": [],96 "version": 1,97 "can_edit": false,98 "can_delete": false,99 "can_recover": false,100 "can_see_hidden_post": false,101 "can_wiki": false,102 "read": true,103 "user_title": null,104 "bookmarked": false,105 "actions_summary": [],106 "moderator": false,107 "admin": false,108 "staff": false,109 "user_id": 9179,110 "hidden": false,111 "trust_level": 2,112 "deleted_at": null,113 "user_deleted": false,114 "edit_reason": null,115 "can_view_edit_history": true,116 "wiki": false,117 "post_url": "/t/sparse-linear-layer/39150/2",118 "can_accept_answer": false,119 "can_unaccept_answer": false,120 "accepted_answer": false,121 "topic_accepted_answer": null122 },123 {124 "id": 167106,125 "name": "Aamir Mirza",126 "username": "Aamir_Mirza",127 "avatar_template": "/user_avatar/discuss.pytorch.org/aamir_mirza/{size}/13336_2.png",128 "created_at": "2020-02-18T23:53:40.176Z",129 "cooked": "<p>here you go , full implementation on git hub for sparse layer <a href=\"https://github.com/numenta/htmpapers/tree/master/arxiv/how_can_we_be_so_dense\" rel=\"nofollow noopener\">https://github.com/numenta/htmpapers/tree/master/arxiv/how_can_we_be_so_dense</a></p>",130 "post_number": 3,131 "post_type": 1,132 "posts_count": 6,133 "updated_at": "2020-02-18T23:53:40.176Z",134 "reply_count": 0,135 "reply_to_post_number": null,136 "quote_count": 0,137 "incoming_link_count": 114,138 "reads": 126,139 "readers_count": 125,140 "score": 595.2,141 "yours": false,142 "topic_id": 39150,143 "topic_slug": "sparse-linear-layer",144 "display_username": "Aamir Mirza",145 "primary_group_name": null,146 "flair_name": null,147 "flair_url": null,148 "flair_bg_color": null,149 "flair_color": null,150 "flair_group_id": null,151 "badges_granted": [],152 "version": 1,153 "can_edit": false,154 "can_delete": false,155 "can_recover": false,156 "can_see_hidden_post": false,157 "can_wiki": false,158 "link_counts": [159 {160 "url": "https://github.com/numenta/htmpapers/tree/master/arxiv/how_can_we_be_so_dense",161 "internal": false,162 "reflection": false,163 "title": "htmpapers/arxiv/how_can_we_be_so_dense at master · numenta/htmpapers · GitHub",164 "clicks": 1513165 }166 ],167 "read": true,168 "user_title": null,169 "bookmarked": false,170 "actions_summary": [],171 "moderator": false,172 "admin": false,173 "staff": false,174 "user_id": 27825,175 "hidden": false,176 "trust_level": 0,177 "deleted_at": null,178 "user_deleted": false,179 "edit_reason": null,180 "can_view_edit_history": true,181 "wiki": false,182 "post_url": "/t/sparse-linear-layer/39150/3",183 "can_accept_answer": false,184 "can_unaccept_answer": false,185 "accepted_answer": false,186 "topic_accepted_answer": null187 },188 {189 "id": 192430,190 "name": "Robloughnan",191 "username": "robloughnan",192 "avatar_template": "/letter_avatar_proxy/v4/letter/r/f9ae1b/{size}.png",193 "created_at": "2020-05-13T18:59:39.984Z",194 "cooked": "<p><a class=\"mention\" href=\"/u/aamir_mirza\">@Aamir_Mirza</a> thanks for sharing. Does this implementation result in saving GPU memory instead of using dense layers? Or is it that it enforces sparsity of the weights for convergence reasons?</p>",195 "post_number": 4,196 "post_type": 1,197 "posts_count": 6,198 "updated_at": "2020-05-13T18:59:39.984Z",199 "reply_count": 0,200 "reply_to_post_number": null,201 "quote_count": 0,202 "incoming_link_count": 184,203 "reads": 115,204 "readers_count": 114,205 "score": 943.0,206 "yours": false,207 "topic_id": 39150,208 "topic_slug": "sparse-linear-layer",209 "display_username": "Robloughnan",210 "primary_group_name": null,211 "flair_name": null,212 "flair_url": null,213 "flair_bg_color": null,214 "flair_color": null,215 "flair_group_id": null,216 "badges_granted": [],217 "version": 1,218 "can_edit": false,219 "can_delete": false,220 "can_recover": false,221 "can_see_hidden_post": false,222 "can_wiki": false,223 "read": true,224 "user_title": null,225 "bookmarked": false,226 "actions_summary": [],227 "moderator": false,228 "admin": false,229 "staff": false,230 "user_id": 30852,231 "hidden": false,232 "trust_level": 1,233 "deleted_at": null,234 "user_deleted": false,235 "edit_reason": null,236 "can_view_edit_history": true,237 "wiki": false,238 "post_url": "/t/sparse-linear-layer/39150/4",239 "can_accept_answer": false,240 "can_unaccept_answer": false,241 "accepted_answer": false,242 "topic_accepted_answer": null243 },244 {245 "id": 246609,246 "name": "Shreyansh Singh",247 "username": "shreyansh26",248 "avatar_template": "/user_avatar/discuss.pytorch.org/shreyansh26/{size}/31436_2.png",249 "created_at": "2020-11-24T05:33:03.621Z",250 "cooked": "<p>Is there any update on this? Any new libraries that implement sparse layer?</p>",251 "post_number": 5,252 "post_type": 1,253 "posts_count": 6,254 "updated_at": "2020-11-24T05:33:03.621Z",255 "reply_count": 1,256 "reply_to_post_number": null,257 "quote_count": 0,258 "incoming_link_count": 114,259 "reads": 91,260 "readers_count": 90,261 "score": 588.2,262 "yours": false,263 "topic_id": 39150,264 "topic_slug": "sparse-linear-layer",265 "display_username": "Shreyansh Singh",266 "primary_group_name": null,267 "flair_name": null,268 "flair_url": null,269 "flair_bg_color": null,270 "flair_color": null,271 "flair_group_id": null,272 "badges_granted": [],273 "version": 1,274 "can_edit": false,275 "can_delete": false,276 "can_recover": false,277 "can_see_hidden_post": false,278 "can_wiki": false,279 "read": true,280 "user_title": null,281 "bookmarked": false,282 "actions_summary": [],283 "moderator": false,284 "admin": false,285 "staff": false,286 "user_id": 39257,287 "hidden": false,288 "trust_level": 1,289 "deleted_at": null,290 "user_deleted": false,291 "edit_reason": null,292 "can_view_edit_history": true,293 "wiki": false,294 "post_url": "/t/sparse-linear-layer/39150/5",295 "can_accept_answer": false,296 "can_unaccept_answer": false,297 "accepted_answer": false,298 "topic_accepted_answer": null299 },300 {301 "id": 324914,302 "name": "Saurabh Kataria",303 "username": "Saurabh_Kataria",304 "avatar_template": "/user_avatar/discuss.pytorch.org/saurabh_kataria/{size}/12442_2.png",305 "created_at": "2022-01-03T21:46:47.448Z",306 "cooked": "<p><a href=\"https://github.com/huggingface/pytorch_block_sparse\" rel=\"noopener nofollow ugc\">https://github.com/huggingface/pytorch_block_sparse</a> is the latest I am aware of but dont know about its performance.</p>",307 "post_number": 6,308 "post_type": 1,309 "posts_count": 6,310 "updated_at": "2022-01-03T21:46:47.448Z",311 "reply_count": 0,312 "reply_to_post_number": 5,313 "quote_count": 0,314 "incoming_link_count": 53,315 "reads": 53,316 "readers_count": 52,317 "score": 275.6,318 "yours": false,319 "topic_id": 39150,320 "topic_slug": "sparse-linear-layer",321 "display_username": "Saurabh Kataria",322 "primary_group_name": null,323 "flair_name": null,324 "flair_url": null,325 "flair_bg_color": null,326 "flair_color": null,327 "flair_group_id": null,328 "badges_granted": [],329 "version": 1,330 "can_edit": false,331 "can_delete": false,332 "can_recover": false,333 "can_see_hidden_post": false,334 "can_wiki": false,335 "link_counts": [336 {337 "url": "https://github.com/huggingface/pytorch_block_sparse",338 "internal": false,339 "reflection": false,340 "title": "GitHub - huggingface/pytorch_block_sparse: Fast Block Sparse Matrices for Pytorch",341 "clicks": 652342 }343 ],344 "read": true,345 "user_title": null,346 "reply_to_user": {347 "id": 39257,348 "username": "shreyansh26",349 "name": "Shreyansh Singh",350 "avatar_template": "/user_avatar/discuss.pytorch.org/shreyansh26/{size}/31436_2.png"351 },352 "bookmarked": false,353 "actions_summary": [],354 "moderator": false,355 "admin": false,356 "staff": false,357 "user_id": 12941,358 "hidden": false,359 "trust_level": 2,360 "deleted_at": null,361 "user_deleted": false,362 "edit_reason": null,363 "can_view_edit_history": true,364 "wiki": false,365 "post_url": "/t/sparse-linear-layer/39150/6",366 "can_accept_answer": false,367 "can_unaccept_answer": false,368 "accepted_answer": false,369 "topic_accepted_answer": null370 }371 ],372 "stream": [373 97781,374 138601,375 167106,376 192430,377 246609,378 324914379 ]380 },381 "timeline_lookup": [382 [383 1,384 2425385 ],386 [387 2,388 2210389 ],390 [391 3,392 2076393 ],394 [395 4,396 1991397 ],398 [399 5,400 1797401 ],402 [403 6,404 1391405 ]406 ],407 "suggested_topics": [408 {409 "fancy_title": "Would calling torch.sort(…, stable=False) ever fail reproducibility?",410 "id": 212956,411 "title": "Would calling torch.sort(..., stable=False) ever fail reproducibility?",412 "slug": "would-calling-torch-sort-stable-false-ever-fail-reproducibility",413 "posts_count": 1,414 "reply_count": 0,415 "highest_post_number": 1,416 "image_url": null,417 "created_at": "2024-11-14T01:08:23.042Z",418 "last_posted_at": "2024-11-14T01:08:23.096Z",419 "bumped": true,420 "bumped_at": "2024-11-14T01:13:57.489Z",421 "archetype": "regular",422 "unseen": false,423 "pinned": false,424 "unpinned": null,425 "visible": true,426 "closed": false,427 "archived": false,428 "bookmarked": null,429 "liked": null,430 "tags_descriptions": {},431 "like_count": 0,432 "views": 70,433 "category_id": 1,434 "featured_link": null,435 "has_accepted_answer": false,436 "posters": [437 {438 "extras": "latest single",439 "description": "Original Poster, Most Recent Poster",440 "user": {441 "id": 80644,442 "username": "Craig_Hicks1",443 "name": "Craig Hicks",444 "avatar_template": "/user_avatar/discuss.pytorch.org/craig_hicks1/{size}/73727_2.png",445 "trust_level": 0446 }447 }448 ]449 },450 {451 "fancy_title": "Pytorch support SM_87 cuda architecture",452 "id": 214103,453 "title": "Pytorch support SM_87 cuda architecture",454 "slug": "pytorch-support-sm-87-cuda-architecture",455 "posts_count": 5,456 "reply_count": 3,457 "highest_post_number": 5,458 "image_url": null,459 "created_at": "2024-12-11T12:01:08.814Z",460 "last_posted_at": "2024-12-12T03:07:12.873Z",461 "bumped": true,462 "bumped_at": "2024-12-12T03:07:12.873Z",463 "archetype": "regular",464 "unseen": false,465 "pinned": false,466 "unpinned": null,467 "visible": true,468 "closed": false,469 "archived": false,470 "bookmarked": null,471 "liked": null,472 "tags_descriptions": {},473 "like_count": 0,474 "views": 479,475 "category_id": 1,476 "featured_link": null,477 "has_accepted_answer": true,478 "posters": [479 {480 "extras": "latest",481 "description": "Original Poster, Most Recent Poster",482 "user": {483 "id": 81448,484 "username": "yajuvendra",485 "name": "Yajuvendra Rawat",486 "avatar_template": "/user_avatar/discuss.pytorch.org/yajuvendra/{size}/74470_2.png",487 "trust_level": 0488 }489 },490 {491 "extras": null,492 "description": "Frequent Poster, Accepted Answer",493 "user": {494 "id": 3534,495 "username": "ptrblck",496 "name": "",497 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",498 "admin": true,499 "moderator": true,500 "trust_level": 2501 }502 }503 ]504 },505 {506 "fancy_title": "Using BatchNorm1d for standardization",507 "id": 214473,508 "title": "Using BatchNorm1d for standardization",509 "slug": "using-batchnorm1d-for-standardization",510 "posts_count": 2,511 "reply_count": 0,512 "highest_post_number": 2,513 "image_url": null,514 "created_at": "2024-12-20T23:17:37.293Z",515 "last_posted_at": "2024-12-20T23:21:22.922Z",516 "bumped": true,517 "bumped_at": "2024-12-20T23:21:22.922Z",518 "archetype": "regular",519 "unseen": false,520 "pinned": false,521 "unpinned": null,522 "visible": true,523 "closed": false,524 "archived": false,525 "bookmarked": null,526 "liked": null,527 "tags_descriptions": {},528 "like_count": 0,529 "views": 35,530 "category_id": 1,531 "featured_link": null,532 "has_accepted_answer": false,533 "posters": [534 {535 "extras": null,536 "description": "Original Poster",537 "user": {538 "id": 65042,539 "username": "ado_sar",540 "name": "ado sar",541 "avatar_template": "/user_avatar/discuss.pytorch.org/ado_sar/{size}/59241_2.png",542 "trust_level": 2543 }544 },545 {546 "extras": "latest",547 "description": "Most Recent Poster",548 "user": {549 "id": 41396,550 "username": "soulitzer",551 "name": "",552 "avatar_template": "/letter_avatar_proxy/v4/letter/s/839c29/{size}.png",553 "trust_level": 2554 }555 }556 ]557 },558 {559 "fancy_title": "Licensing for PyTorch Word Usage in Books",560 "id": 215808,561 "title": "Licensing for PyTorch Word Usage in Books",562 "slug": "licensing-for-pytorch-word-usage-in-books",563 "posts_count": 10,564 "reply_count": 7,565 "highest_post_number": 10,566 "image_url": null,567 "created_at": "2025-01-24T07:25:14.106Z",568 "last_posted_at": "2025-01-28T07:02:20.511Z",569 "bumped": true,570 "bumped_at": "2025-01-28T07:02:20.511Z",571 "archetype": "regular",572 "unseen": false,573 "pinned": false,574 "unpinned": null,575 "visible": true,576 "closed": false,577 "archived": false,578 "bookmarked": null,579 "liked": null,580 "tags_descriptions": {},581 "like_count": 0,582 "views": 96,583 "category_id": 1,584 "featured_link": null,585 "has_accepted_answer": false,586 "posters": [587 {588 "extras": "latest",589 "description": "Original Poster, Most Recent Poster",590 "user": {591 "id": 82282,592 "username": "orangeeducationpvt",593 "name": "Orange Education Pvt Ltd",594 "avatar_template": "/letter_avatar_proxy/v4/letter/o/97f17d/{size}.png",595 "trust_level": 0596 }597 },598 {599 "extras": null,600 "description": "Frequent Poster",601 "user": {602 "id": 3534,603 "username": "ptrblck",604 "name": "",605 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",606 "admin": true,607 "moderator": true,608 "trust_level": 2609 }610 },611 {612 "extras": null,613 "description": "Frequent Poster",614 "user": {615 "id": 75469,616 "username": "Matt_White",617 "name": "Matt White",618 "avatar_template": "/user_avatar/discuss.pytorch.org/matt_white/{size}/69693_2.png",619 "admin": true,620 "trust_level": 1621 }622 }623 ]624 },625 {626 "fancy_title": "How to add value to tensor multiple times by index",627 "id": 212382,628 "title": "How to add value to tensor multiple times by index",629 "slug": "how-to-add-value-to-tensor-multiple-times-by-index",630 "posts_count": 3,631 "reply_count": 0,632 "highest_post_number": 3,633 "image_url": null,634 "created_at": "2024-10-31T20:08:52.361Z",635 "last_posted_at": "2024-11-06T07:14:05.689Z",636 "bumped": true,637 "bumped_at": "2024-11-06T07:14:05.689Z",638 "archetype": "regular",639 "unseen": false,640 "pinned": false,641 "unpinned": null,642 "visible": true,643 "closed": false,644 "archived": false,645 "bookmarked": null,646 "liked": null,647 "tags_descriptions": {},648 "like_count": 1,649 "views": 181,650 "category_id": 1,651 "featured_link": null,652 "has_accepted_answer": false,653 "posters": [654 {655 "extras": "latest",656 "description": "Original Poster, Most Recent Poster",657 "user": {658 "id": 80623,659 "username": "dronnet1",660 "name": "Андрей Медведев",661 "avatar_template": "/user_avatar/discuss.pytorch.org/dronnet1/{size}/73708_2.png",662 "trust_level": 1663 }664 },665 {666 "extras": null,667 "description": "Frequent Poster",668 "user": {669 "id": 72430,670 "username": "Eduardo_Lawson",671 "name": "Eduardo Lawson da Silva",672 "avatar_template": "/user_avatar/discuss.pytorch.org/eduardo_lawson/{size}/66899_2.png",673 "trust_level": 2674 }675 }676 ]677 }678 ],679 "tags_descriptions": {},680 "fancy_title": "Sparse Linear layer",681 "id": 39150,682 "title": "Sparse Linear layer",683 "posts_count": 6,684 "created_at": "2019-03-06T20:49:37.877Z",685 "views": 7761,686 "reply_count": 1,687 "like_count": 2,688 "last_posted_at": "2022-01-03T21:46:47.448Z",689 "visible": true,690 "closed": false,691 "archived": false,692 "has_summary": false,693 "archetype": "regular",694 "slug": "sparse-linear-layer",695 "category_id": 1,696 "word_count": 193,697 "deleted_at": null,698 "user_id": 16402,699 "featured_link": null,700 "pinned_globally": false,701 "pinned_at": null,702 "pinned_until": null,703 "image_url": null,704 "slow_mode_seconds": 0,705 "draft": null,706 "draft_key": "topic_39150",707 "draft_sequence": null,708 "unpinned": null,709 "pinned": false,710 "current_post_number": 1,711 "highest_post_number": 6,712 "deleted_by": null,713 "actions_summary": [714 {715 "id": 4,716 "count": 0,717 "hidden": false,718 "can_act": false719 },720 {721 "id": 8,722 "count": 0,723 "hidden": false,724 "can_act": false725 },726 {727 "id": 10,728 "count": 0,729 "hidden": false,730 "can_act": false731 },732 {733 "id": 7,734 "count": 0,735 "hidden": false,736 "can_act": false737 }738 ],739 "chunk_size": 20,740 "bookmarked": false,741 "topic_timer": null,742 "message_bus_last_id": 0,743 "participant_count": 6,744 "show_read_indicator": false,745 "thumbnails": null,746 "slow_mode_enabled_until": null,747 "can_vote": false,748 "vote_count": 0,749 "user_voted": false,750 "discourse_zendesk_plugin_zendesk_id": null,751 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",752 "details": {753 "can_edit": false,754 "notification_level": 1,755 "participants": [756 {757 "id": 9179,758 "username": "Kale-ab_Tessera",759 "name": "Kale Ab Tessera",760 "avatar_template": "/user_avatar/discuss.pytorch.org/kale-ab_tessera/{size}/11407_2.png",761 "post_count": 1,762 "primary_group_name": null,763 "flair_name": null,764 "flair_url": null,765 "flair_color": null,766 "flair_bg_color": null,767 "flair_group_id": null,768 "trust_level": 2769 },770 {771 "id": 12941,772 "username": "Saurabh_Kataria",773 "name": "Saurabh Kataria",774 "avatar_template": "/user_avatar/discuss.pytorch.org/saurabh_kataria/{size}/12442_2.png",775 "post_count": 1,776 "primary_group_name": null,777 "flair_name": null,778 "flair_url": null,779 "flair_color": null,780 "flair_bg_color": null,781 "flair_group_id": null,782 "trust_level": 2783 },784 {785 "id": 16402,786 "username": "Albert65",787 "name": "",788 "avatar_template": "/user_avatar/discuss.pytorch.org/albert65/{size}/10175_2.png",789 "post_count": 1,790 "primary_group_name": null,791 "flair_name": null,792 "flair_url": null,793 "flair_color": null,794 "flair_bg_color": null,795 "flair_group_id": null,796 "trust_level": 1797 },798 {799 "id": 27825,800 "username": "Aamir_Mirza",801 "name": "Aamir Mirza",802 "avatar_template": "/user_avatar/discuss.pytorch.org/aamir_mirza/{size}/13336_2.png",803 "post_count": 1,804 "primary_group_name": null,805 "flair_name": null,806 "flair_url": null,807 "flair_color": null,808 "flair_bg_color": null,809 "flair_group_id": null,810 "trust_level": 0811 },812 {813 "id": 30852,814 "username": "robloughnan",815 "name": "Robloughnan",816 "avatar_template": "/letter_avatar_proxy/v4/letter/r/f9ae1b/{size}.png",817 "post_count": 1,818 "primary_group_name": null,819 "flair_name": null,820 "flair_url": null,821 "flair_color": null,822 "flair_bg_color": null,823 "flair_group_id": null,824 "trust_level": 1825 },826 {827 "id": 39257,828 "username": "shreyansh26",829 "name": "Shreyansh Singh",830 "avatar_template": "/user_avatar/discuss.pytorch.org/shreyansh26/{size}/31436_2.png",831 "post_count": 1,832 "primary_group_name": null,833 "flair_name": null,834 "flair_url": null,835 "flair_color": null,836 "flair_bg_color": null,837 "flair_group_id": null,838 "trust_level": 1839 }840 ],841 "created_by": {842 "id": 16402,843 "username": "Albert65",844 "name": "",845 "avatar_template": "/user_avatar/discuss.pytorch.org/albert65/{size}/10175_2.png"846 },847 "last_poster": {848 "id": 12941,849 "username": "Saurabh_Kataria",850 "name": "Saurabh Kataria",851 "avatar_template": "/user_avatar/discuss.pytorch.org/saurabh_kataria/{size}/12442_2.png"852 },853 "links": [854 {855 "url": "https://github.com/numenta/htmpapers/tree/master/arxiv/how_can_we_be_so_dense",856 "title": "htmpapers/arxiv/how_can_we_be_so_dense at master · numenta/htmpapers · GitHub",857 "internal": false,858 "attachment": false,859 "reflection": false,860 "clicks": 1513,861 "user_id": 27825,862 "domain": "github.com",863 "root_domain": "github.com"864 },865 {866 "url": "https://github.com/huggingface/pytorch_block_sparse",867 "title": "GitHub - huggingface/pytorch_block_sparse: Fast Block Sparse Matrices for Pytorch",868 "internal": false,869 "attachment": false,870 "reflection": false,871 "clicks": 652,872 "user_id": 12941,873 "domain": "github.com",874 "root_domain": "github.com"875 }876 ]877 },878 "bookmarks": []879 },880 {881 "post_stream": {882 "posts": [883 {884 "id": 317085,885 "name": "Jun Wan",886 "username": "Jun_Wan",887 "avatar_template": "/user_avatar/discuss.pytorch.org/jun_wan/{size}/34806_2.png",888 "created_at": "2021-11-13T19:59:55.771Z",889 "cooked": "<p>Hi, I compared two tests:</p>\n<ol>\n<li>\n<p>resnet20 on cifar10 with privacy-engine, the clipping norm is set to 10M. This should be equivalent to not doing clipping at all.</p>\n</li>\n<li>\n<p>resnet20 on cifar10 without privacy-engine (noise-multiplier is set as 0), with exactly the same parameters as example 1.</p>\n</li>\n</ol>\n<p>Test 2 soon reached 92% accuracy while test 1 struggled to reach 85%. I then wrote another test where we created two models for the two above tests and made them train on the same data (with the same trainloader) simultaneously. Model 1 has optimizer 1 which is attached to a privacy-engine, while model 2 has optimizer 2 which is just a normal SGD optimizer.</p>\n<p>The code looks something like this:</p>\n<p>loss1.backward()<br>\nloss2.backward()</p>\n<p>optimizer1.step()<br>\noptimizer2.step()</p>\n<p>Before we call the step() functions, the param.grad are exactly the same between the two models. However, after we called the step() functions, there are approximately a 3% difference between the param.grad of the two models.</p>\n<p>Is this because pytorch’s default way of computing gradient is different from opacus even when the clipping value is 10 million? Or is it because of accuracy loss during opacus computations?</p>",890 "post_number": 1,891 "post_type": 1,892 "posts_count": 7,893 "updated_at": "2021-11-13T20:26:32.906Z",894 "reply_count": 1,895 "reply_to_post_number": null,896 "quote_count": 0,897 "incoming_link_count": 111,898 "reads": 24,899 "readers_count": 23,900 "score": 564.8,901 "yours": false,902 "topic_id": 136817,903 "topic_slug": "opacus-with-large-clipping-norm-performs-differently-from-normal-network",904 "display_username": "Jun Wan",905 "primary_group_name": null,906 "flair_name": null,907 "flair_url": null,908 "flair_bg_color": null,909 "flair_color": null,910 "flair_group_id": null,911 "badges_granted": [],912 "version": 2,913 "can_edit": false,914 "can_delete": false,915 "can_recover": false,916 "can_see_hidden_post": false,917 "can_wiki": false,918 "read": true,919 "user_title": null,920 "bookmarked": false,921 "actions_summary": [],922 "moderator": false,923 "admin": false,924 "staff": false,925 "user_id": 50693,926 "hidden": false,927 "trust_level": 0,928 "deleted_at": null,929 "user_deleted": false,930 "edit_reason": null,931 "can_view_edit_history": true,932 "wiki": false,933 "post_url": "/t/opacus-with-large-clipping-norm-performs-differently-from-normal-network/136817/1",934 "can_accept_answer": false,935 "can_unaccept_answer": false,936 "accepted_answer": false,937 "topic_accepted_answer": null,938 "can_vote": false939 },940 {941 "id": 317278,942 "name": "Ilya Mironov",943 "username": "ilyamironov",944 "avatar_template": "/user_avatar/discuss.pytorch.org/ilyamironov/{size}/32160_2.png",945 "created_at": "2021-11-15T06:28:12.272Z",946 "cooked": "<p>Are you sure you use the same exact network in both cases? The “canonical” Resnet20 includes batch normalization, which is incompatible with DP-SGD.</p>",947 "post_number": 2,948 "post_type": 1,949 "posts_count": 7,950 "updated_at": "2021-11-15T06:28:12.272Z",951 "reply_count": 1,952 "reply_to_post_number": null,953 "quote_count": 0,954 "incoming_link_count": 1,955 "reads": 24,956 "readers_count": 23,957 "score": 14.8,958 "yours": false,959 "topic_id": 136817,960 "topic_slug": "opacus-with-large-clipping-norm-performs-differently-from-normal-network",961 "display_username": "Ilya Mironov",962 "primary_group_name": null,963 "flair_name": null,964 "flair_url": null,965 "flair_bg_color": null,966 "flair_color": null,967 "flair_group_id": null,968 "badges_granted": [],969 "version": 1,970 "can_edit": false,971 "can_delete": false,972 "can_recover": false,973 "can_see_hidden_post": false,974 "can_wiki": false,975 "read": true,976 "user_title": null,977 "bookmarked": false,978 "actions_summary": [],979 "moderator": false,980 "admin": false,981 "staff": false,982 "user_id": 39907,983 "hidden": false,984 "trust_level": 1,985 "deleted_at": null,986 "user_deleted": false,987 "edit_reason": null,988 "can_view_edit_history": true,989 "wiki": false,990 "post_url": "/t/opacus-with-large-clipping-norm-performs-differently-from-normal-network/136817/2",991 "can_accept_answer": false,992 "can_unaccept_answer": false,993 "accepted_answer": false,994 "topic_accepted_answer": null995 },996 {997 "id": 317411,998 "name": "Jun Wan",999 "username": "Jun_Wan",1000 "avatar_template": "/user_avatar/discuss.pytorch.org/jun_wan/{size}/34806_2.png",1001 "created_at": "2021-11-15T21:41:16.414Z",1002 "cooked": "<p>Yes. We called the opacus’ convert_batchnorm_modules() function for both models. All the batchnorm layers are converted to groupnorm layers.</p>\n<p>We also use model2.load_state_dict(copy.deepcopy(model1.state_dict())) at the beginning to make sure they start with the same network parameters.</p>",1003 "post_number": 3,1004 "post_type": 1,1005 "posts_count": 7,1006 "updated_at": "2021-11-15T21:41:16.414Z",1007 "reply_count": 1,1008 "reply_to_post_number": 2,1009 "quote_count": 0,1010 "incoming_link_count": 1,1011 "reads": 21,1012 "readers_count": 20,1013 "score": 14.2,1014 "yours": false,1015 "topic_id": 136817,1016 "topic_slug": "opacus-with-large-clipping-norm-performs-differently-from-normal-network",1017 "display_username": "Jun Wan",1018 "primary_group_name": null,1019 "flair_name": null,1020 "flair_url": null,1021 "flair_bg_color": null,1022 "flair_color": null,1023 "flair_group_id": null,1024 "badges_granted": [],1025 "version": 1,1026 "can_edit": false,1027 "can_delete": false,1028 "can_recover": false,1029 "can_see_hidden_post": false,1030 "can_wiki": false,1031 "read": true,1032 "user_title": null,1033 "reply_to_user": {1034 "id": 39907,1035 "username": "ilyamironov",1036 "name": "Ilya Mironov",1037 "avatar_template": "/user_avatar/discuss.pytorch.org/ilyamironov/{size}/32160_2.png"1038 },1039 "bookmarked": false,1040 "actions_summary": [],1041 "moderator": false,1042 "admin": false,1043 "staff": false,1044 "user_id": 50693,1045 "hidden": false,1046 "trust_level": 0,1047 "deleted_at": null,1048 "user_deleted": false,1049 "edit_reason": null,1050 "can_view_edit_history": true,1051 "wiki": false,1052 "post_url": "/t/opacus-with-large-clipping-norm-performs-differently-from-normal-network/136817/3",1053 "can_accept_answer": false,1054 "can_unaccept_answer": false,1055 "accepted_answer": false,1056 "topic_accepted_answer": null1057 },1058 {1059 "id": 319261,1060 "name": "Karthik Prasad",1061 "username": "karthikprasad",1062 "avatar_template": "/user_avatar/discuss.pytorch.org/karthikprasad/{size}/32155_2.png",1063 "created_at": "2021-11-26T23:32:36.789Z",1064 "cooked": "<p>Hi <a class=\"mention\" href=\"/u/jun_wan\">@Jun_Wan</a>, is it possible your noise multiplier was non-zero in your first case? If yes, do you mind sharing a notebook with the above issue? This will help us debug further.</p>",1065 "post_number": 4,1066 "post_type": 1,1067 "posts_count": 7,1068 "updated_at": "2021-11-26T23:32:36.789Z",1069 "reply_count": 0,1070 "reply_to_post_number": 3,1071 "quote_count": 0,1072 "incoming_link_count": 2,1073 "reads": 19,1074 "readers_count": 18,1075 "score": 28.8,1076 "yours": false,1077 "topic_id": 136817,1078 "topic_slug": "opacus-with-large-clipping-norm-performs-differently-from-normal-network",1079 "display_username": "Karthik Prasad",1080 "primary_group_name": null,1081 "flair_name": null,1082 "flair_url": null,1083 "flair_bg_color": null,1084 "flair_color": null,1085 "flair_group_id": null,1086 "badges_granted": [],1087 "version": 1,1088 "can_edit": false,1089 "can_delete": false,1090 "can_recover": false,1091 "can_see_hidden_post": false,1092 "can_wiki": false,1093 "read": true,1094 "user_title": null,1095 "reply_to_user": {1096 "id": 50693,1097 "username": "Jun_Wan",1098 "name": "Jun Wan",1099 "avatar_template": "/user_avatar/discuss.pytorch.org/jun_wan/{size}/34806_2.png"1100 },1101 "bookmarked": false,1102 "actions_summary": [1103 {1104 "id": 2,1105 "count": 11106 }1107 ],1108 "moderator": false,1109 "admin": false,1110 "staff": false,1111 "user_id": 39902,1112 "hidden": false,1113 "trust_level": 2,1114 "deleted_at": null,1115 "user_deleted": false,1116 "edit_reason": null,1117 "can_view_edit_history": true,1118 "wiki": false,1119 "post_url": "/t/opacus-with-large-clipping-norm-performs-differently-from-normal-network/136817/4",1120 "can_accept_answer": false,1121 "can_unaccept_answer": false,1122 "accepted_answer": false,1123 "topic_accepted_answer": null1124 },1125 {1126 "id": 320967,1127 "name": "",1128 "username": "ashkan_software",1129 "avatar_template": "/letter_avatar_proxy/v4/letter/a/a587f6/{size}.png",1130 "created_at": "2021-12-07T23:08:48.957Z",1131 "cooked": "<p>Hi <a class=\"mention\" href=\"/u/jun_wan\">@Jun_Wan</a>. Has your issue been resolved? If not, do you mind sharing a notebook so we can look into this?</p>",1132 "post_number": 5,1133 "post_type": 1,1134 "posts_count": 7,1135 "updated_at": "2021-12-07T23:08:48.957Z",1136 "reply_count": 0,1137 "reply_to_post_number": null,1138 "quote_count": 0,1139 "incoming_link_count": 1,1140 "reads": 15,1141 "readers_count": 14,1142 "score": 8.0,1143 "yours": false,1144 "topic_id": 136817,1145 "topic_slug": "opacus-with-large-clipping-norm-performs-differently-from-normal-network",1146 "display_username": "",1147 "primary_group_name": null,1148 "flair_name": null,1149 "flair_url": null,1150 "flair_bg_color": null,1151 "flair_color": null,1152 "flair_group_id": null,1153 "badges_granted": [],1154 "version": 1,1155 "can_edit": false,1156 "can_delete": false,1157 "can_recover": false,1158 "can_see_hidden_post": false,1159 "can_wiki": false,1160 "read": true,1161 "user_title": null,1162 "bookmarked": false,1163 "actions_summary": [],1164 "moderator": false,1165 "admin": false,1166 "staff": false,1167 "user_id": 41987,1168 "hidden": false,1169 "trust_level": 2,1170 "deleted_at": null,1171 "user_deleted": false,1172 "edit_reason": null,1173 "can_view_edit_history": true,1174 "wiki": false,1175 "post_url": "/t/opacus-with-large-clipping-norm-performs-differently-from-normal-network/136817/5",1176 "can_accept_answer": false,1177 "can_unaccept_answer": false,1178 "accepted_answer": false,1179 "topic_accepted_answer": null1180 },1181 {1182 "id": 321141,1183 "name": "Jun Wan",1184 "username": "Jun_Wan",1185 "avatar_template": "/user_avatar/discuss.pytorch.org/jun_wan/{size}/34806_2.png",1186 "created_at": "2021-12-08T23:19:44.184Z",1187 "cooked": "<p>Hi, sorry. For some reasons, the previous notification emails went to the trash folder. I just noticed them today.</p>\n<p>I posted our test code on <a href=\"https://github.com/junwan0224/test-code\" class=\"inline-onebox\" rel=\"noopener nofollow ugc\">GitHub - junwan0224/test-code</a>. We used the compare.py file compare the gradients. One model has privacy engine while the other does not. We feed them the exact same data, but the resulted gradients are different.</p>\n<p>For a previous question: yes, our noise multiplier is set as zero. Please let me know if there is any other question. Thank you for helping!</p>",1188 "post_number": 6,1189 "post_type": 1,1190 "posts_count": 7,1191 "updated_at": "2021-12-08T23:19:44.184Z",1192 "reply_count": 1,1193 "reply_to_post_number": null,1194 "quote_count": 0,1195 "incoming_link_count": 0,1196 "reads": 15,1197 "readers_count": 14,1198 "score": 8.0,1199 "yours": false,1200 "topic_id": 136817,