Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 458568,7 "name": "Arkapravo Ghosh",8 "username": "Arkapravo_Ghosh",9 "avatar_template": "/user_avatar/discuss.pytorch.org/arkapravo_ghosh/{size}/73737_2.png",10 "created_at": "2024-11-02T19:37:13.435Z",11 "cooked": "<p>I have defined a custom linear class named ‘Linear’ (nn.Module) as follows:</p>\n<pre><code class=\"lang-auto\">import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom prot_map import *\n\nclass Linear(nn.Module):\n def __init__(\n self,\n in_features,\n out_features,\n bias=True,\n name=None,\n num_prot = 50,\n C=16\n ):\n super(Linear, self).__init__()\n self.in_features = in_features\n self.out_features = out_features\n self.name = name\n self.num_prot = num_prot\n self.C = C\n\n # Define weight and bias parameters\n self.weight = nn.Parameter(torch.Tensor(out_features, in_features), requires_grad=True)\n self.bias = nn.Parameter(torch.Tensor(out_features), requires_grad=True) if bias else None\n self.prototype = nn.Parameter(torch.Tensor(num_prot, in_features), requires_grad=True)\n nn.init.normal_(self.prototype, mean=0, std=0.01)\n\n def append_name(self, postfix):\n self.name += postfix\n\n def forward(self, input):\n # Get dimensions\n batch, N, D = input.shape\n K, D_prot = self.prototype.shape\n assert D == D_prot, \"Input and prototype dimensions must match\"\n\n # Split input and prototype across C codespaces\n input_split = input.view(batch, N, self.C, D // self.C) # Shape: (batch, N, C, D//C)\n prototype_split = self.prototype.view(K, self.C, D // self.C) # Shape: (K, C, D//C)\n\n # Initialize lists to store mapped inputs and probabilities across codespaces\n x_map = []\n \n for c in range(self.C):\n \n # Assume input_split[:, :, c, :] has shape (batch, N, D//C) and prototype_split[:, c, :] has shape (K, D//C)\n input_data = input_split[:, :, c, :] # Shape: (batch, N, D//C)\n prototypes = prototype_split[:, c, :] # Shape: (K, D//C)\n\n # Compute distances for each codespace independently\n distances = torch.cdist(input_split[:, :, c, :], prototype_split[:, c, :], p=2) #shape = (batch, N, K)\n epsilon = 1e-8\n prob = F.softmax(1 / (distances + epsilon), dim = -1) # Shape: (batch, N, K)\n assert prob.shape == (batch, N, K), \"Incorrect probability tensor shape\"\n\n # Map input to prototypes for each codespace\n mapped_input = prob @ prototypes # Shape: (batch, N, D//C)\n x_map.append(mapped_input)\n\n # Concatenate mapped inputs and prob_matrix from all codespaces\n mapped_input = torch.cat(x_map, dim=-1) # Shape: (batch, N, D)\n\n output = F.linear(mapped_input, self.weight, self.bias) # Shape: (batch, N, out_features)\n\n return output\n</code></pre>\n<p>I am trying to map the inputs to learnable prototypes as outlined in the code above. However, when I use this Linear class during training, I find that prototype.grad is None by using this code:</p>\n<pre><code class=\"lang-auto\">for name, module in model.named_modules():\n if isinstance(module, Linear):\n if module.prototype.grad is not None:\n print(f\"Gradient of prototype parameter in {name}:\", module.prototype.grad)\n else:\n print(f\"No gradient for prototype parameter in {name}!\")\n</code></pre>\n<p>Suggest me how to ensure that the gradients are propagated and where am I going wrong</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 2,15 "updated_at": "2024-11-02T19:37:13.435Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 8,20 "reads": 8,21 "readers_count": 7,22 "score": 41.6,23 "yours": false,24 "topic_id": 212455,25 "topic_slug": "no-gradient-found-for-a-parameter-in-custom-linear-class",26 "display_username": "Arkapravo Ghosh",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 80655,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/no-gradient-found-for-a-parameter-in-custom-linear-class/212455/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 458633,64 "name": "",65 "username": "ptrblck",66 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",67 "created_at": "2024-11-04T14:49:26.100Z",68 "cooked": "<p>I cannot reproduce any issue and see a valid gradient in <code>.prototype.grad</code>:</p>\n<pre data-code-wrap=\"python\"><code class=\"lang-python\">lin = Linear(10, 10, C=10)\nx = torch.randn(1, 10, 10)\n\nout = lin(x)\nout.mean().backward()\n\nprint(lin.prototype.grad.abs().sum())\n# tensor(40590.8516)\n</code></pre>\n<p>but I also don’t know how you are using this model.</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 2,72 "updated_at": "2024-11-04T14:49:26.100Z",73 "reply_count": 0,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 1,77 "reads": 5,78 "readers_count": 4,79 "score": 6.0,80 "yours": false,81 "topic_id": 212455,82 "topic_slug": "no-gradient-found-for-a-parameter-in-custom-linear-class",83 "display_username": "",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 3534,105 "hidden": false,106 "trust_level": 2,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/no-gradient-found-for-a-parameter-in-custom-linear-class/212455/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 }118 ],119 "stream": [120 458568,121 458633122 ]123 },124 "timeline_lookup": [125 [126 1,127 357128 ],129 [130 2,131 355132 ]133 ],134 "suggested_topics": [135 {136 "fancy_title": "Building llama-cpp-python with CUDA support fails due to GLIBC version incompatibility",137 "id": 221001,138 "title": "Building llama-cpp-python with CUDA support fails due to GLIBC version incompatibility",139 "slug": "building-llama-cpp-python-with-cuda-support-fails-due-to-glibc-version-incompatibility",140 "posts_count": 1,141 "reply_count": 0,142 "highest_post_number": 1,143 "image_url": null,144 "created_at": "2025-06-23T15:26:19.501Z",145 "last_posted_at": "2025-06-23T15:26:19.546Z",146 "bumped": true,147 "bumped_at": "2025-06-23T15:26:19.546Z",148 "archetype": "regular",149 "unseen": false,150 "pinned": false,151 "unpinned": null,152 "visible": true,153 "closed": false,154 "archived": false,155 "bookmarked": null,156 "liked": null,157 "tags_descriptions": {},158 "like_count": 0,159 "views": 164,160 "category_id": 5,161 "featured_link": null,162 "has_accepted_answer": false,163 "posters": [164 {165 "extras": "latest single",166 "description": "Original Poster, Most Recent Poster",167 "user": {168 "id": 84787,169 "username": "Oba",170 "name": "Oba Ozai",171 "avatar_template": "/user_avatar/discuss.pytorch.org/oba/{size}/77442_2.png",172 "trust_level": 1173 }174 }175 ]176 },177 {178 "fancy_title": "Encoder-Bottleneck-Decoder Unet Based Semantic Segmentation",179 "id": 214096,180 "title": "Encoder-Bottleneck-Decoder Unet Based Semantic Segmentation",181 "slug": "encoder-bottleneck-decoder-unet-based-semantic-segmentation",182 "posts_count": 3,183 "reply_count": 1,184 "highest_post_number": 3,185 "image_url": null,186 "created_at": "2024-12-11T10:13:08.305Z",187 "last_posted_at": "2024-12-18T06:52:53.050Z",188 "bumped": true,189 "bumped_at": "2024-12-18T06:52:53.050Z",190 "archetype": "regular",191 "unseen": false,192 "pinned": false,193 "unpinned": null,194 "visible": true,195 "closed": false,196 "archived": false,197 "bookmarked": null,198 "liked": null,199 "tags_descriptions": {},200 "like_count": 1,201 "views": 314,202 "category_id": 5,203 "featured_link": null,204 "has_accepted_answer": false,205 "posters": [206 {207 "extras": "latest",208 "description": "Original Poster, Most Recent Poster",209 "user": {210 "id": 81422,211 "username": "Idrees11",212 "name": "Idrees Bhat",213 "avatar_template": "/user_avatar/discuss.pytorch.org/idrees11/{size}/74448_2.png",214 "trust_level": 1215 }216 },217 {218 "extras": null,219 "description": "Frequent Poster",220 "user": {221 "id": 18088,222 "username": "KFrank",223 "name": "K. Frank",224 "avatar_template": "/letter_avatar_proxy/v4/letter/k/ecb155/{size}.png",225 "trust_level": 2226 }227 }228 ]229 },230 {231 "fancy_title": "CNN with Custom Convolutions, loss NAN",232 "id": 214137,233 "title": "CNN with Custom Convolutions, loss NAN",234 "slug": "cnn-with-custom-convolutions-loss-nan",235 "posts_count": 1,236 "reply_count": 0,237 "highest_post_number": 1,238 "image_url": "https://discuss.pytorch.org/uploads/default/original/3X/a/5/a5183b79ade7a3d44fca8ca047f0d878a1f250b8.png",239 "created_at": "2024-12-12T06:44:53.771Z",240 "last_posted_at": "2024-12-12T06:44:53.827Z",241 "bumped": true,242 "bumped_at": "2024-12-12T06:44:53.827Z",243 "archetype": "regular",244 "unseen": false,245 "pinned": false,246 "unpinned": null,247 "visible": true,248 "closed": false,249 "archived": false,250 "bookmarked": null,251 "liked": null,252 "tags_descriptions": {},253 "like_count": 0,254 "views": 82,255 "category_id": 5,256 "featured_link": null,257 "has_accepted_answer": false,258 "posters": [259 {260 "extras": "latest single",261 "description": "Original Poster, Most Recent Poster",262 "user": {263 "id": 81464,264 "username": "sharom_m",265 "name": "",266 "avatar_template": "/user_avatar/discuss.pytorch.org/sharom_m/{size}/73009_2.png",267 "trust_level": 1268 }269 }270 ]271 },272 {273 "fancy_title": "Gradient and the tensor dtype inconsistencies",274 "id": 219066,275 "title": "Gradient and the tensor dtype inconsistencies",276 "slug": "gradient-and-the-tensor-dtype-inconsistencies",277 "posts_count": 4,278 "reply_count": 1,279 "highest_post_number": 4,280 "image_url": null,281 "created_at": "2025-04-14T14:07:44.627Z",282 "last_posted_at": "2025-04-15T14:13:39.474Z",283 "bumped": true,284 "bumped_at": "2025-04-15T14:13:39.474Z",285 "archetype": "regular",286 "unseen": false,287 "pinned": false,288 "unpinned": null,289 "visible": true,290 "closed": false,291 "archived": false,292 "bookmarked": null,293 "liked": null,294 "tags_descriptions": {},295 "like_count": 0,296 "views": 126,297 "category_id": 5,298 "featured_link": null,299 "has_accepted_answer": false,300 "posters": [301 {302 "extras": null,303 "description": "Original Poster",304 "user": {305 "id": 29433,306 "username": "cltexe",307 "name": "Omer Faruk Soylemez",308 "avatar_template": "/user_avatar/discuss.pytorch.org/cltexe/{size}/41817_2.png",309 "trust_level": 1310 }311 },312 {313 "extras": null,314 "description": "Frequent Poster",315 "user": {316 "id": 77908,317 "username": "mycul",318 "name": "",319 "avatar_template": "/user_avatar/discuss.pytorch.org/mycul/{size}/72394_2.png",320 "trust_level": 2321 }322 },323 {324 "extras": "latest",325 "description": "Most Recent Poster",326 "user": {327 "id": 3534,328 "username": "ptrblck",329 "name": "",330 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",331 "admin": true,332 "moderator": true,333 "trust_level": 2334 }335 }336 ]337 },338 {339 "fancy_title": "Fine tuning pretrained RestNet for grayscale image classification",340 "id": 221018,341 "title": "Fine tuning pretrained RestNet for grayscale image classification",342 "slug": "fine-tuning-pretrained-restnet-for-grayscale-image-classification",343 "posts_count": 2,344 "reply_count": 0,345 "highest_post_number": 2,346 "image_url": null,347 "created_at": "2025-06-24T11:03:10.058Z",348 "last_posted_at": "2025-06-25T14:22:46.038Z",349 "bumped": true,350 "bumped_at": "2025-06-25T14:22:46.038Z",351 "archetype": "regular",352 "unseen": false,353 "pinned": false,354 "unpinned": null,355 "visible": true,356 "closed": false,357 "archived": false,358 "bookmarked": null,359 "liked": null,360 "tags_descriptions": {},361 "like_count": 0,362 "views": 70,363 "category_id": 5,364 "featured_link": null,365 "has_accepted_answer": false,366 "posters": [367 {368 "extras": null,369 "description": "Original Poster",370 "user": {371 "id": 84808,372 "username": "abir",373 "name": "",374 "avatar_template": "/user_avatar/discuss.pytorch.org/abir/{size}/77461_2.png",375 "trust_level": 1376 }377 },378 {379 "extras": "latest",380 "description": "Most Recent Poster",381 "user": {382 "id": 3534,383 "username": "ptrblck",384 "name": "",385 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",386 "admin": true,387 "moderator": true,388 "trust_level": 2389 }390 }391 ]392 }393 ],394 "tags_descriptions": {},395 "fancy_title": "No gradient found for a parameter in custom Linear class",396 "id": 212455,397 "title": "No gradient found for a parameter in custom Linear class",398 "posts_count": 2,399 "created_at": "2024-11-02T19:37:13.384Z",400 "views": 42,401 "reply_count": 0,402 "like_count": 0,403 "last_posted_at": "2024-11-04T14:49:26.100Z",404 "visible": true,405 "closed": false,406 "archived": false,407 "has_summary": false,408 "archetype": "regular",409 "slug": "no-gradient-found-for-a-parameter-in-custom-linear-class",410 "category_id": 5,411 "word_count": 469,412 "deleted_at": null,413 "user_id": 80655,414 "featured_link": null,415 "pinned_globally": false,416 "pinned_at": null,417 "pinned_until": null,418 "image_url": null,419 "slow_mode_seconds": 0,420 "draft": null,421 "draft_key": "topic_212455",422 "draft_sequence": null,423 "unpinned": null,424 "pinned": false,425 "current_post_number": 1,426 "highest_post_number": 2,427 "deleted_by": null,428 "actions_summary": [429 {430 "id": 4,431 "count": 0,432 "hidden": false,433 "can_act": false434 },435 {436 "id": 8,437 "count": 0,438 "hidden": false,439 "can_act": false440 },441 {442 "id": 10,443 "count": 0,444 "hidden": false,445 "can_act": false446 },447 {448 "id": 7,449 "count": 0,450 "hidden": false,451 "can_act": false452 }453 ],454 "chunk_size": 20,455 "bookmarked": false,456 "topic_timer": null,457 "message_bus_last_id": 0,458 "participant_count": 2,459 "show_read_indicator": false,460 "thumbnails": null,461 "slow_mode_enabled_until": null,462 "can_vote": false,463 "vote_count": 0,464 "user_voted": false,465 "discourse_zendesk_plugin_zendesk_id": null,466 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",467 "details": {468 "can_edit": false,469 "notification_level": 1,470 "participants": [471 {472 "id": 3534,473 "username": "ptrblck",474 "name": "",475 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",476 "post_count": 1,477 "primary_group_name": null,478 "flair_name": null,479 "flair_url": null,480 "flair_color": null,481 "flair_bg_color": null,482 "flair_group_id": null,483 "admin": true,484 "moderator": true,485 "trust_level": 2486 },487 {488 "id": 80655,489 "username": "Arkapravo_Ghosh",490 "name": "Arkapravo Ghosh",491 "avatar_template": "/user_avatar/discuss.pytorch.org/arkapravo_ghosh/{size}/73737_2.png",492 "post_count": 1,493 "primary_group_name": null,494 "flair_name": null,495 "flair_url": null,496 "flair_color": null,497 "flair_bg_color": null,498 "flair_group_id": null,499 "trust_level": 1500 }501 ],502 "created_by": {503 "id": 80655,504 "username": "Arkapravo_Ghosh",505 "name": "Arkapravo Ghosh",506 "avatar_template": "/user_avatar/discuss.pytorch.org/arkapravo_ghosh/{size}/73737_2.png"507 },508 "last_poster": {509 "id": 3534,510 "username": "ptrblck",511 "name": "",512 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"513 }514 },515 "bookmarks": []516 },517 {518 "post_stream": {519 "posts": [520 {521 "id": 458625,522 "name": "yuan wentao",523 "username": "yuan_wentao",524 "avatar_template": "/user_avatar/discuss.pytorch.org/yuan_wentao/{size}/41715_2.png",525 "created_at": "2024-11-04T12:49:06.619Z",526 "cooked": "<p>I am conducting a DDP parallel training task, and I only want to perform validation in the main process. I used <code>torch.distributed.barrier</code> to prevent non-main processes to avoid NCCL timeout. However, I found that torch.distributed.barrier consumes an additional approximately 4GB of GPU memory (from 93GB to 97GB). Can someone help explain why this happens?</p>",527 "post_number": 1,528 "post_type": 1,529 "posts_count": 1,530 "updated_at": "2024-11-04T12:49:06.619Z",531 "reply_count": 0,532 "reply_to_post_number": null,533 "quote_count": 0,534 "incoming_link_count": 10,535 "reads": 5,536 "readers_count": 4,537 "score": 51.0,538 "yours": false,539 "topic_id": 212502,540 "topic_slug": "torch-distributed-barrier-occupies-additional-cuda-memory",541 "display_username": "yuan wentao",542 "primary_group_name": null,543 "flair_name": null,544 "flair_url": null,545 "flair_bg_color": null,546 "flair_color": null,547 "flair_group_id": null,548 "badges_granted": [],549 "version": 1,550 "can_edit": false,551 "can_delete": false,552 "can_recover": false,553 "can_see_hidden_post": false,554 "can_wiki": false,555 "read": true,556 "user_title": null,557 "bookmarked": false,558 "actions_summary": [],559 "moderator": false,560 "admin": false,561 "staff": false,562 "user_id": 48526,563 "hidden": false,564 "trust_level": 1,565 "deleted_at": null,566 "user_deleted": false,567 "edit_reason": null,568 "can_view_edit_history": true,569 "wiki": false,570 "post_url": "/t/torch-distributed-barrier-occupies-additional-cuda-memory/212502/1",571 "can_accept_answer": false,572 "can_unaccept_answer": false,573 "accepted_answer": false,574 "topic_accepted_answer": null,575 "can_vote": false576 }577 ],578 "stream": [579 458625580 ]581 },582 "timeline_lookup": [583 [584 1,585 355586 ]587 ],588 "suggested_topics": [589 {590 "fancy_title": "How to handle RAM OOM in DDP?",591 "id": 219245,592 "title": "How to handle RAM OOM in DDP?",593 "slug": "how-to-handle-ram-oom-in-ddp",594 "posts_count": 4,595 "reply_count": 2,596 "highest_post_number": 4,597 "image_url": null,598 "created_at": "2025-04-19T08:33:38.442Z",599 "last_posted_at": "2025-04-19T14:08:42.350Z",600 "bumped": true,601 "bumped_at": "2025-04-19T14:08:42.350Z",602 "archetype": "regular",603 "unseen": false,604 "pinned": false,605 "unpinned": null,606 "visible": true,607 "closed": false,608 "archived": false,609 "bookmarked": null,610 "liked": null,611 "tags_descriptions": {},612 "like_count": 0,613 "views": 128,614 "category_id": 12,615 "featured_link": null,616 "has_accepted_answer": false,617 "posters": [618 {619 "extras": null,620 "description": "Original Poster",621 "user": {622 "id": 82744,623 "username": "Maria_Djeblahi",624 "name": "Maria Djeblahi",625 "avatar_template": "/user_avatar/discuss.pytorch.org/maria_djeblahi/{size}/75704_2.png",626 "trust_level": 1627 }628 },629 {630 "extras": "latest",631 "description": "Most Recent Poster",632 "user": {633 "id": 3534,634 "username": "ptrblck",635 "name": "",636 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",637 "admin": true,638 "moderator": true,639 "trust_level": 2640 }641 }642 ]643 },644 {645 "fancy_title": "Difference between ProcessGroup and Backend classes",646 "id": 213483,647 "title": "Difference between ProcessGroup and Backend classes",648 "slug": "difference-between-processgroup-and-backend-classes",649 "posts_count": 2,650 "reply_count": 0,651 "highest_post_number": 2,652 "image_url": null,653 "created_at": "2024-11-26T18:15:06.946Z",654 "last_posted_at": "2024-12-02T16:30:22.440Z",655 "bumped": true,656 "bumped_at": "2024-12-02T16:30:22.440Z",657 "archetype": "regular",658 "unseen": false,659 "pinned": false,660 "unpinned": null,661 "visible": true,662 "closed": false,663 "archived": false,664 "bookmarked": null,665 "liked": null,666 "tags_descriptions": {},667 "like_count": 1,668 "views": 165,669 "category_id": 12,670 "featured_link": null,671 "has_accepted_answer": false,672 "posters": [673 {674 "extras": null,675 "description": "Original Poster",676 "user": {677 "id": 76967,678 "username": "nathanbrown-Arm",679 "name": "Nathan Brown",680 "avatar_template": "/letter_avatar_proxy/v4/letter/n/9dc877/{size}.png",681 "trust_level": 1682 }683 },684 {685 "extras": "latest",686 "description": "Most Recent Poster",687 "user": {688 "id": 54320,689 "username": "fduwjj",690 "name": "Hugo",691 "avatar_template": "/user_avatar/discuss.pytorch.org/fduwjj/{size}/47855_2.png",692 "trust_level": 2693 }694 }695 ]696 },697 {698 "fancy_title": "FSDP clarifying questions",699 "id": 216159,700 "title": "FSDP clarifying questions",701 "slug": "fsdp-clarifying-questions",702 "posts_count": 1,703 "reply_count": 0,704 "highest_post_number": 1,705 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/7/e/7e4f544ff3e53365869083460811a12c04e40f1b_2_1024x462.png",706 "created_at": "2025-02-03T02:48:51.398Z",707 "last_posted_at": "2025-02-03T02:48:51.437Z",708 "bumped": true,709 "bumped_at": "2025-02-03T02:48:51.437Z",710 "archetype": "regular",711 "unseen": false,712 "pinned": false,713 "unpinned": null,714 "visible": true,715 "closed": false,716 "archived": false,717 "bookmarked": null,718 "liked": null,719 "tags_descriptions": {},720 "like_count": 0,721 "views": 128,722 "category_id": 12,723 "featured_link": null,724 "has_accepted_answer": false,725 "posters": [726 {727 "extras": "latest single",728 "description": "Original Poster, Most Recent Poster",729 "user": {730 "id": 82447,731 "username": "Samir_Char",732 "name": "Samir Char",733 "avatar_template": "/user_avatar/discuss.pytorch.org/samir_char/{size}/75136_2.png",734 "trust_level": 0735 }736 }737 ]738 },739 {740 "fancy_title": "Memory error on ONE GPU destribution on the CPU befor moving the data",741 "id": 219118,742 "title": "Memory error on ONE GPU destribution on the CPU befor moving the data",743 "slug": "memory-error-on-one-gpu-destribution-on-the-cpu-befor-moving-the-data",744 "posts_count": 4,745 "reply_count": 2,746 "highest_post_number": 4,747 "image_url": null,748 "created_at": "2025-04-15T17:54:47.025Z",749 "last_posted_at": "2025-04-15T23:18:33.367Z",750 "bumped": true,751 "bumped_at": "2025-04-15T23:18:33.367Z",752 "archetype": "regular",753 "unseen": false,754 "pinned": false,755 "unpinned": null,756 "visible": true,757 "closed": false,758 "archived": false,759 "bookmarked": null,760 "liked": null,761 "tags_descriptions": {},762 "like_count": 0,763 "views": 61,764 "category_id": 12,765 "featured_link": null,766 "has_accepted_answer": false,767 "posters": [768 {769 "extras": null,770 "description": "Original Poster",771 "user": {772 "id": 82744,773 "username": "Maria_Djeblahi",774 "name": "Maria Djeblahi",775 "avatar_template": "/user_avatar/discuss.pytorch.org/maria_djeblahi/{size}/75704_2.png",776 "trust_level": 1777 }778 },779 {780 "extras": "latest",781 "description": "Most Recent Poster",782 "user": {783 "id": 3534,784 "username": "ptrblck",785 "name": "",786 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",787 "admin": true,788 "moderator": true,789 "trust_level": 2790 }791 }792 ]793 },794 {795 "fancy_title": "Ddp training and eval question",796 "id": 221823,797 "title": "Ddp training and eval question",798 "slug": "ddp-training-and-eval-question",799 "posts_count": 3,800 "reply_count": 1,801 "highest_post_number": 3,802 "image_url": null,803 "created_at": "2025-07-26T07:56:48.862Z",804 "last_posted_at": "2025-07-26T21:43:09.321Z",805 "bumped": true,806 "bumped_at": "2025-07-26T21:43:09.321Z",807 "archetype": "regular",808 "unseen": false,809 "pinned": false,810 "unpinned": null,811 "visible": true,812 "closed": false,813 "archived": false,814 "bookmarked": null,815 "liked": null,816 "tags_descriptions": {},817 "like_count": 2,818 "views": 38,819 "category_id": 12,820 "featured_link": null,821 "has_accepted_answer": false,822 "posters": [823 {824 "extras": "latest",825 "description": "Original Poster, Most Recent Poster",826 "user": {827 "id": 85233,828 "username": "Julius_Lee",829 "name": "Julius Lee",830 "avatar_template": "/user_avatar/discuss.pytorch.org/julius_lee/{size}/77782_2.png",831 "trust_level": 0832 }833 },834 {835 "extras": null,836 "description": "Frequent Poster",837 "user": {838 "id": 39542,839 "username": "H-Huang",840 "name": "Howard Huang",841 "avatar_template": "/user_avatar/discuss.pytorch.org/h-huang/{size}/35598_2.png",842 "trust_level": 2843 }844 }845 ]846 }847 ],848 "tags_descriptions": {},849 "fancy_title": "Torch.distributed.barrier occupies additional CUDA memory",850 "id": 212502,851 "title": "Torch.distributed.barrier occupies additional CUDA memory",852 "posts_count": 1,853 "created_at": "2024-11-04T12:49:06.441Z",854 "views": 99,855 "reply_count": 0,856 "like_count": 0,857 "last_posted_at": "2024-11-04T12:49:06.619Z",858 "visible": true,859 "closed": false,860 "archived": false,861 "has_summary": false,862 "archetype": "regular",863 "slug": "torch-distributed-barrier-occupies-additional-cuda-memory",864 "category_id": 12,865 "word_count": 59,866 "deleted_at": null,867 "user_id": 48526,868 "featured_link": null,869 "pinned_globally": false,870 "pinned_at": null,871 "pinned_until": null,872 "image_url": null,873 "slow_mode_seconds": 0,874 "draft": null,875 "draft_key": "topic_212502",876 "draft_sequence": null,877 "unpinned": null,878 "pinned": false,879 "current_post_number": 1,880 "highest_post_number": 1,881 "deleted_by": null,882 "actions_summary": [883 {884 "id": 4,885 "count": 0,886 "hidden": false,887 "can_act": false888 },889 {890 "id": 8,891 "count": 0,892 "hidden": false,893 "can_act": false894 },895 {896 "id": 10,897 "count": 0,898 "hidden": false,899 "can_act": false900 },901 {902 "id": 7,903 "count": 0,904 "hidden": false,905 "can_act": false906 }907 ],908 "chunk_size": 20,909 "bookmarked": false,910 "topic_timer": null,911 "message_bus_last_id": 0,912 "participant_count": 1,913 "show_read_indicator": false,914 "thumbnails": null,915 "slow_mode_enabled_until": null,916 "can_vote": false,917 "vote_count": 0,918 "user_voted": false,919 "discourse_zendesk_plugin_zendesk_id": null,920 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",921 "details": {922 "can_edit": false,923 "notification_level": 1,924 "participants": [925 {926 "id": 48526,927 "username": "yuan_wentao",928 "name": "yuan wentao",929 "avatar_template": "/user_avatar/discuss.pytorch.org/yuan_wentao/{size}/41715_2.png",930 "post_count": 1,931 "primary_group_name": null,932 "flair_name": null,933 "flair_url": null,934 "flair_color": null,935 "flair_bg_color": null,936 "flair_group_id": null,937 "trust_level": 1938 }939 ],940 "created_by": {941 "id": 48526,942 "username": "yuan_wentao",943 "name": "yuan wentao",944 "avatar_template": "/user_avatar/discuss.pytorch.org/yuan_wentao/{size}/41715_2.png"945 },946 "last_poster": {947 "id": 48526,948 "username": "yuan_wentao",949 "name": "yuan wentao",950 "avatar_template": "/user_avatar/discuss.pytorch.org/yuan_wentao/{size}/41715_2.png"951 }952 },953 "bookmarks": []954 },955 {956 "post_stream": {957 "posts": [958 {959 "id": 458624,960 "name": "Joel Mwanja",961 "username": "mwanjajoel",962 "avatar_template": "/user_avatar/discuss.pytorch.org/mwanjajoel/{size}/63523_2.png",963 "created_at": "2024-11-04T12:01:31.381Z",964 "cooked": "<p>Hello I am training an AI on detecting cyber security threats but my training accuracy and epoch are the same and this is bothering me. how do I fix it?</p>\n<p>Here is my code.</p>\n<pre><code class=\"lang-auto\">import pandas as pd\nfrom sklearn.preprocessing import StandardScaler\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as functional\nfrom torch.utils.data import DataLoader, TensorDataset\nimport torch.optim as optim\nfrom torchmetrics import Accuracy\n\n# load the pre-processed data\ntrain_df = pd.read_csv('datasets/cybersecurity_threats/labelled_train.csv')\ntest_df = pd.read_csv('datasets/cybersecurity_threats/labelled_test.csv')\nval_df = pd.read_csv('datasets/cybersecurity_threats/labelled_validation.csv')\n\n# Show the first 10 rows of the dataset\nval_df.head(10)\n\n # Convert the training dataset into Pytorch tensors\n\n selected_columns = ['processId', 'threadId', 'parentProcessId', 'userId', 'mountNamespace', 'argsNum', 'returnValue']\n\n# Get the features and change them to numpy_array\nfeatures = train_df[selected_columns].to_numpy()\nlabels = train_df['sus_label'].to_numpy()\n\nval_features = val_df[selected_columns].to_numpy()\nval_label = val_df['sus_label'].to_numpy()\n\ntest_features = test_df[selected_columns].to_numpy()\ntest_label = test_df['sus_label'].to_numpy()\n\n\n# Create the features tensor\nfeatures_tensor = torch.tensor(features, dtype=torch.float32)\n\n# Create the labels tensor\nlabels_tensor = torch.tensor(labels, dtype=torch.float32)\nval_labels_tensor = torch.tensor(val_label, dtype=torch.float32)\ntest_labels_tensor = torch.tensor(test_label, dtype=torch.float32)\n\n# Create the test features tensor\ntest_features = torch.tensor(test_features, dtype=torch.float32)\n\n\n# Create the validation features tensor\nval_features = torch.tensor(val_features, dtype=torch.float32)\n\n# Combine the features and labels into a tensor dataset\ntrain_dataset = TensorDataset(features_tensor, labels_tensor)\nval_dataset = TensorDataset(val_features, val_labels_tensor)\ntest_dataset = TensorDataset(test_features, test_labels_tensor)\n\n# load the training dataset into the DataLoader\nbatch_size = 5\ntrain_loader = DataLoader(train_dataset, batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size, shuffle=True)\ntest_loader = DataLoader(test_dataset, batch_size, shuffle=True)\n\n# Create the model\nclass CyberSecurityModel(nn.Module):\n def __init__(self, input_size, hidden_size, output_size):\n super(CyberSecurityModel, self).__init__()\n\n self.fc1 = nn.Linear(input_size, hidden_size)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(p=0.02)\n self.fc2 = nn.Linear(hidden_size, output_size)\n self.sigmoid = nn.Sigmoid()\n \n # forward\n def forward(self, x):\n x = self.fc1(x)\n x = self.relu(x)\n x = self.fc2(x)\n\n return self.sigmoid(x)\n#initiliase the model, criterion and optimizer \ninput_size = len(selected_columns)\nhidden_size = 512\noutput_size = 1\n\nmodel = CyberSecurityModel(input_size, hidden_size, output_size)\ndevice = torch.device(\"cpu\")\n\ncriterion = nn.BCELoss()\nlearning_rate = 0.0001\n# momentum = 0.7\n\n\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n# initialize the accuracy metrics for training, validation and testing\ntrain_accuracy = Accuracy(task=\"binary\").to(device)\ntest_accuracy = Accuracy(task=\"binary\").to(device)\nval_accuracy = Accuracy(task=\"binary\").to(device)\n\n# create the training loop\nnumber_of_epochs = 50\nmodel = model.to(device)\n\nfor epoch in range(number_of_epochs):\n model.train()\n running_loss = 0\n\n # Reset the training accuracy metric\n train_accuracy.reset()\n\n for i, data in enumerate(train_loader):\n inputs, labels = data\n\n inputs = inputs.to(device)\n labels = labels.to(device)\n\n # set the optimizer to zero gradient\n optimizer.zero_grad()\n\n #forward pass\n outputs = model(inputs).squeeze()\n loss = criterion(outputs, labels)\n loss.backward()\n optimizer.step()\n\n # Update the training accuracy\n train_accuracy.update(outputs, labels.int())\n\n # Track the loss\n running_loss += loss.item()\n \n # Compute the average training accuracy for the epoch\n avg_train_accuracy = train_accuracy.compute()\n\n print(f'Epoch [{epoch+1} / {number_of_epochs}], '\n f'Training Loss: { running_loss / len(train_loader):.2f}, '\n f'Training Accuracy: {avg_train_accuracy * 100:.2f}%,'\n )\n</code></pre>",965 "post_number": 1,966 "post_type": 1,967 "posts_count": 1,968 "updated_at": "2024-11-04T12:01:31.381Z",969 "reply_count": 0,970 "reply_to_post_number": null,971 "quote_count": 0,972 "incoming_link_count": 4,973 "reads": 4,974 "readers_count": 3,975 "score": 20.8,976 "yours": false,977 "topic_id": 212501,978 "topic_slug": "my-training-loss-and-training-accuracy-are-the-same-for-each-epoch-how-do-i-fix-it",979 "display_username": "Joel Mwanja",980 "primary_group_name": null,981 "flair_name": null,982 "flair_url": null,983 "flair_bg_color": null,984 "flair_color": null,985 "flair_group_id": null,986 "badges_granted": [],987 "version": 1,988 "can_edit": false,989 "can_delete": false,990 "can_recover": false,991 "can_see_hidden_post": false,992 "can_wiki": false,993 "read": true,994 "user_title": null,995 "bookmarked": false,996 "actions_summary": [],997 "moderator": false,998 "admin": false,999 "staff": false,1000 "user_id": 69040,1001 "hidden": false,1002 "trust_level": 0,1003 "deleted_at": null,1004 "user_deleted": false,1005 "edit_reason": null,1006 "can_view_edit_history": true,1007 "wiki": false,1008 "post_url": "/t/my-training-loss-and-training-accuracy-are-the-same-for-each-epoch-how-do-i-fix-it/212501/1",1009 "can_accept_answer": false,1010 "can_unaccept_answer": false,1011 "accepted_answer": false,1012 "topic_accepted_answer": null,1013 "can_vote": false1014 }1015 ],1016 "stream": [1017 4586241018 ]1019 },1020 "timeline_lookup": [1021 [1022 1,1023 3551024 ]1025 ],1026 "suggested_topics": [1027 {1028 "fancy_title": "Improved PyTorch Models in Minutes with Perforated Backpropagation",1029 "id": 219592,1030 "title": "Improved PyTorch Models in Minutes with Perforated Backpropagation",1031 "slug": "improved-pytorch-models-in-minutes-with-perforated-backpropagation",1032 "posts_count": 1,1033 "reply_count": 0,1034 "highest_post_number": 1,1035 "image_url": null,1036 "created_at": "2025-04-29T13:05:13.507Z",1037 "last_posted_at": "2025-04-29T13:05:13.558Z",1038 "bumped": true,1039 "bumped_at": "2025-04-29T13:05:13.558Z",1040 "archetype": "regular",1041 "unseen": false,1042 "pinned": false,1043 "unpinned": null,1044 "visible": true,1045 "closed": false,1046 "archived": false,1047 "bookmarked": null,1048 "liked": null,1049 "tags_descriptions": {},1050 "like_count": 1,1051 "views": 69,1052 "category_id": 33,1053 "featured_link": null,1054 "has_accepted_answer": false,1055 "posters": [1056 {1057 "extras": "latest single",1058 "description": "Original Poster, Most Recent Poster",1059 "user": {1060 "id": 73736,1061 "username": "PerforatedAI",1062 "name": "Rorry Brenner",1063 "avatar_template": "/user_avatar/discuss.pytorch.org/perforatedai/{size}/76845_2.png",1064 "trust_level": 11065 }1066 }1067 ]1068 },1069 {1070 "fancy_title": "Torchzero - modular optimization library",1071 "id": 221520,1072 "title": "Torchzero - modular optimization library",1073 "slug": "torchzero-modular-optimization-library",1074 "posts_count": 1,1075 "reply_count": 0,1076 "highest_post_number": 1,1077 "image_url": null,1078 "created_at": "2025-07-14T19:56:41.890Z",1079 "last_posted_at": "2025-07-14T19:56:41.932Z",1080 "bumped": true,1081 "bumped_at": "2025-07-16T04:38:44.736Z",1082 "archetype": "regular",1083 "unseen": false,1084 "pinned": false,1085 "unpinned": null,1086 "visible": true,1087 "closed": false,1088 "archived": false,1089 "bookmarked": null,1090 "liked": null,1091 "tags_descriptions": {},1092 "like_count": 1,1093 "views": 68,1094 "category_id": 33,1095 "featured_link": null,1096 "has_accepted_answer": false,1097 "posters": [1098 {1099 "extras": "latest single",1100 "description": "Original Poster, Most Recent Poster",1101 "user": {1102 "id": 75871,1103 "username": "qq-me",1104 "name": "Ivan Nikishev",1105 "avatar_template": "/user_avatar/discuss.pytorch.org/qq-me/{size}/70055_2.png",1106 "trust_level": 21107 }1108 }1109 ]1110 },1111 {1112 "fancy_title": "AnyModal – A Framework for Multimodal LLMs",1113 "id": 213081,1114 "title": "AnyModal – A Framework for Multimodal LLMs",1115 "slug": "anymodal-a-framework-for-multimodal-llms",1116 "posts_count": 1,1117 "reply_count": 0,1118 "highest_post_number": 1,1119 "image_url": null,1120 "created_at": "2024-11-17T18:21:02.644Z",1121 "last_posted_at": "2024-11-17T18:21:02.683Z",1122 "bumped": true,1123 "bumped_at": "2024-11-17T18:21:02.683Z",1124 "archetype": "regular",1125 "unseen": false,1126 "pinned": false,1127 "unpinned": null,1128 "visible": true,1129 "closed": false,1130 "archived": false,1131 "bookmarked": null,1132 "liked": null,1133 "tags_descriptions": {},1134 "like_count": 0,1135 "views": 70,1136 "category_id": 33,1137 "featured_link": null,1138 "has_accepted_answer": false,1139 "posters": [1140 {1141 "extras": "latest single",1142 "description": "Original Poster, Most Recent Poster",1143 "user": {1144 "id": 80967,1145 "username": "ritabratamaiti",1146 "name": "Ritabrata Maiti",1147 "avatar_template": "/user_avatar/discuss.pytorch.org/ritabratamaiti/{size}/74049_2.png",1148 "trust_level": 11149 }1150 }1151 ]1152 },1153 {1154 "fancy_title": "Where to Post for Tensor Subclass Support in SKA?",1155 "id": 217295,1156 "title": "Where to Post for Tensor Subclass Support in SKA?",1157 "slug": "where-to-post-for-tensor-subclass-support-in-ska",1158 "posts_count": 1,1159 "reply_count": 0,1160 "highest_post_number": 1,1161 "image_url": null,1162 "created_at": "2025-02-28T13:44:24.686Z",1163 "last_posted_at": "2025-02-28T13:44:24.726Z",1164 "bumped": true,1165 "bumped_at": "2025-02-28T13:44:24.726Z",1166 "archetype": "regular",1167 "unseen": false,1168 "pinned": false,1169 "unpinned": null,1170 "visible": true,1171 "closed": false,1172 "archived": false,1173 "bookmarked": null,1174 "liked": null,1175 "tags_descriptions": {},1176 "like_count": 0,1177 "views": 20,1178 "category_id": 33,1179 "featured_link": null,1180 "has_accepted_answer": false,1181 "posters": [1182 {1183 "extras": "latest single",1184 "description": "Original Poster, Most Recent Poster",1185 "user": {1186 "id": 82994,1187 "username": "BouarfaMahi",1188 "name": "Bouarfa Mahi",1189 "avatar_template": "/user_avatar/discuss.pytorch.org/bouarfamahi/{size}/75923_2.png",1190 "trust_level": 01191 }1192 }1193 ]1194 },1195 {1196 "fancy_title": "Jupyter notebook repository for self-learning",1197 "id": 222622,1198 "title": "Jupyter notebook repository for self-learning",1199 "slug": "jupyter-notebook-repository-for-self-learning",1200 "posts_count": 1,