Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 220412,7 "name": "Yun Lee",8 "username": "Yun_Lee",9 "avatar_template": "/user_avatar/discuss.pytorch.org/yun_lee/{size}/13932_2.png",10 "created_at": "2020-08-12T22:00:45.932Z",11 "cooked": "<p>I was to attack a model with adversarial examples<br>\nand got IndexError : tuple index out of range during backward pass</p>\n<pre><code class=\"lang-auto\"> 35 # Calculate gradients of model in backward pass\n---> 36 loss.backward()\n 37 \n 38 # Collect datagrad\n\n~/anaconda3/envs/yoon/lib/python3.6/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)\n 116 products. Defaults to ``False``.\n 117 \"\"\"\n--> 118 torch.autograd.backward(self, gradient, retain_graph, create_graph)\n 119 \n 120 def register_hook(self, hook):\n\n~/anaconda3/envs/yoon/lib/python3.6/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\n 91 Variable._execution_engine.run_backward(\n 92 tensors, grad_tensors, retain_graph, create_graph,\n---> 93 allow_unreachable=True) # allow_unreachable flag\n 94 \n 95 \n\n~/anaconda3/envs/yoon/lib/python3.6/site-packages/torch/autograd/function.py in apply(self, *args)\n 75 \n 76 def apply(self, *args):\n---> 77 return self._forward_cls.backward(self, *args)\n 78 \n 79 \n\n~/Block-wise-Scrambled-Image-Recognition/shakedrop.py in backward(ctx, grad_output)\n 25 @staticmethod\n 26 def backward(ctx, grad_output):\n---> 27 gate = ctx.saved_tensors[0]\n 28 if gate.item() == 0:\n 29 beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_(0, 1)\n\nIndexError: tuple index out of range\n</code></pre>\n<p>but I can get loss value</p>\n<pre><code class=\"lang-auto\">fgsm\noutput is tensor([[-1.0937, -2.6928, 1.2549, 2.1249, 1.4102, 2.1192, 1.3507, -0.2068,\n -1.5742, -2.8067]], device='cuda:0', grad_fn=<AddmmBackward>)\ntarget is tensor([3], device='cuda:0')\ntensor(-2.1249, device='cuda:0', grad_fn=<NllLossBackward>)\n\n</code></pre>\n<p>test code was this ( batch size is 1 to check if original input is classified well )</p>\n<pre><code class=\"lang-python\">def test( model, device, test_loader, epsilon, attack_name ):\n \n\n # Accuracy counter\n correct = 0\n adv_examples = []\n print(attack_name)\n # Loop over all examples in test set\n for data, target in test_loader:\n\n \n # Send the data and label to the device\n data, target = data.to(device), target.to(device)\n\n # Set requires_grad attribute of tensor. Important for Attack\n data.requires_grad = True\n\n # Forward pass the data through the model\n output = model(data)\n print(\"output is\",output)\n print(\"target is\", target)\n init_pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n \n\n # If the initial prediction is wrong, dont bother attacking, just move on\n if init_pred.item() != target.item():\n continue\n # Calculate the loss\n loss = F.nll_loss(output, target)\n print(loss)\n\n # Zero all existing gradients\n model.zero_grad()\nan\n # Calculate gradients of model in backward pass\n loss.backward()\n\n # Collect datagrad\n data_grad = data.grad.data\n # Call attack\n</code></pre>\n<p>dataset is cifar10<br>\nand I used model pyramidnet</p>\n<pre><code class=\"lang-python\">class ShakeBasicBlock(nn.Module):\n\n def __init__(self, in_ch, out_ch, stride=1, p_shakedrop=1.0):\n super(ShakeBasicBlock, self).__init__()\n self.downsampled = stride == 2\n self.branch = self._make_branch(in_ch, out_ch, stride=stride)\n self.shortcut = not self.downsampled and None or nn.AvgPool2d(2)\n self.shake_drop = ShakeDrop(p_shakedrop)\n\n def forward(self, x):\n h = self.branch(x)\n h = self.shake_drop(h)\n h0 = x if not self.downsampled else self.shortcut(x)\n pad_zero = Variable(torch.zeros(h0.size(0), h.size(1) - h0.size(1), h0.size(2), h0.size(3)).float()).cuda()\n h0 = torch.cat([h0, pad_zero], dim=1)\n\n return h + h0\n\n def _make_branch(self, in_ch, out_ch, stride=1):\n return nn.Sequential(\n nn.BatchNorm2d(in_ch),\n nn.Conv2d(in_ch, out_ch, 3, padding=1, stride=stride, bias=False),\n nn.BatchNorm2d(out_ch),\n nn.ReLU(inplace=True),\n nn.Conv2d(out_ch, out_ch, 3, padding=1, stride=1, bias=False),\n nn.BatchNorm2d(out_ch))\n\n\nclass ShakePyramidNet(nn.Module):\n\n def __init__(self, depth=110, alpha=270, label=10):\n super(ShakePyramidNet, self).__init__()\n in_ch = 16\n # for BasicBlock\n n_units = (depth - 2) // 6\n in_chs = [in_ch] + [in_ch + math.ceil((alpha / (3 * n_units)) * (i + 1)) for i in range(3 * n_units)]\n block = ShakeBasicBlock\n\n self.in_chs, self.u_idx = in_chs, 0\n self.ps_shakedrop = [1 - (1.0 - (0.5 / (3 * n_units)) * (i + 1)) for i in range(3 * n_units)]\n\n self.c_in = nn.Conv2d(3, in_chs[0], 3, padding=1)\n self.bn_in = nn.BatchNorm2d(in_chs[0])\n self.layer1 = self._make_layer(n_units, block, 1)\n self.layer2 = self._make_layer(n_units, block, 2)\n self.layer3 = self._make_layer(n_units, block, 2)\n self.bn_out = nn.BatchNorm2d(in_chs[-1])\n self.fc_out = nn.Linear(in_chs[-1], label)\n\n # Initialize paramters\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n elif isinstance(m, nn.BatchNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n elif isinstance(m, nn.Linear):\n m.bias.data.zero_()\n\n def forward(self, x):\n h = self.bn_in(self.c_in(x))\n feature = h\n h = self.layer1(h)\n h = self.layer2(h)\n h = self.layer3(h)\n h = F.relu(self.bn_out(h))\n h = F.avg_pool2d(h, 8)\n h = h.view(h.size(0), -1)\n h = self.fc_out(h)\n return h\n\n def _make_layer(self, n_units, block, stride=1):\n layers = []\n for i in range(int(n_units)):\n layers.append(block(self.in_chs[self.u_idx], self.in_chs[self.u_idx+1],\n stride, self.ps_shakedrop[self.u_idx]))\n self.u_idx, stride = self.u_idx + 1, 1\n return nn.Sequential(*layers)\n</code></pre>\n<p>Do you know how to solve this problem?</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 4,15 "updated_at": "2020-08-12T22:00:45.932Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 1231,20 "reads": 28,21 "readers_count": 27,22 "score": 6160.6,23 "yours": false,24 "topic_id": 92605,25 "topic_slug": "indexerror-tuple-index-out-of-range-during-backward-pass",26 "display_username": "Yun Lee",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 35478,48 "hidden": false,49 "trust_level": 1,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/indexerror-tuple-index-out-of-range-during-backward-pass/92605/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 220424,64 "name": "Alban D",65 "username": "albanD",66 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",67 "created_at": "2020-08-12T23:51:54.180Z",68 "cooked": "<p>The error code points to the custom function in <code>shakedrop.py</code>.<br>\nIt seems that you try to index the saved_tensors but you did you save any Tensor during the forward?</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 4,72 "updated_at": "2020-08-12T23:51:54.180Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 9,77 "reads": 27,78 "readers_count": 26,79 "score": 55.4,80 "yours": false,81 "topic_id": 92605,82 "topic_slug": "indexerror-tuple-index-out-of-range-during-backward-pass",83 "display_username": "Alban D",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 211,105 "hidden": false,106 "trust_level": 4,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/indexerror-tuple-index-out-of-range-during-backward-pass/92605/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 220757,120 "name": "Yun Lee",121 "username": "Yun_Lee",122 "avatar_template": "/user_avatar/discuss.pytorch.org/yun_lee/{size}/13932_2.png",123 "created_at": "2020-08-14T04:27:36.652Z",124 "cooked": "<p>oh why I didn’t see that sentence…!<br>\nI should see that function first.<br>\nthank you for the answer</p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 4,128 "updated_at": "2020-08-14T04:35:17.011Z",129 "reply_count": 0,130 "reply_to_post_number": 2,131 "quote_count": 0,132 "incoming_link_count": 5,133 "reads": 26,134 "readers_count": 25,135 "score": 75.2,136 "yours": false,137 "topic_id": 92605,138 "topic_slug": "indexerror-tuple-index-out-of-range-during-backward-pass",139 "display_username": "Yun Lee",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 2,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "read": true,154 "user_title": null,155 "reply_to_user": {156 "id": 211,157 "username": "albanD",158 "name": "Alban D",159 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png"160 },161 "bookmarked": false,162 "actions_summary": [163 {164 "id": 2,165 "count": 1166 }167 ],168 "moderator": false,169 "admin": false,170 "staff": false,171 "user_id": 35478,172 "hidden": false,173 "trust_level": 1,174 "deleted_at": null,175 "user_deleted": false,176 "edit_reason": null,177 "can_view_edit_history": true,178 "wiki": false,179 "post_url": "/t/indexerror-tuple-index-out-of-range-during-backward-pass/92605/3",180 "can_accept_answer": false,181 "can_unaccept_answer": false,182 "accepted_answer": false,183 "topic_accepted_answer": null184 },185 {186 "id": 374408,187 "name": "Jiasongd",188 "username": "jiasongd",189 "avatar_template": "/user_avatar/discuss.pytorch.org/jiasongd/{size}/54782_2.png",190 "created_at": "2022-11-13T02:01:58.807Z",191 "cooked": "<p>I have the same problem can you tell me How to modify it??? thank you.</p>",192 "post_number": 4,193 "post_type": 1,194 "posts_count": 4,195 "updated_at": "2022-11-13T02:01:58.807Z",196 "reply_count": 0,197 "reply_to_post_number": null,198 "quote_count": 0,199 "incoming_link_count": 2,200 "reads": 6,201 "readers_count": 5,202 "score": 11.2,203 "yours": false,204 "topic_id": 92605,205 "topic_slug": "indexerror-tuple-index-out-of-range-during-backward-pass",206 "display_username": "Jiasongd",207 "primary_group_name": null,208 "flair_name": null,209 "flair_url": null,210 "flair_bg_color": null,211 "flair_color": null,212 "flair_group_id": null,213 "badges_granted": [],214 "version": 1,215 "can_edit": false,216 "can_delete": false,217 "can_recover": false,218 "can_see_hidden_post": false,219 "can_wiki": false,220 "read": true,221 "user_title": null,222 "bookmarked": false,223 "actions_summary": [],224 "moderator": false,225 "admin": false,226 "staff": false,227 "user_id": 60897,228 "hidden": false,229 "trust_level": 0,230 "deleted_at": null,231 "user_deleted": false,232 "edit_reason": null,233 "can_view_edit_history": true,234 "wiki": false,235 "post_url": "/t/indexerror-tuple-index-out-of-range-during-backward-pass/92605/4",236 "can_accept_answer": false,237 "can_unaccept_answer": false,238 "accepted_answer": false,239 "topic_accepted_answer": null240 }241 ],242 "stream": [243 220412,244 220424,245 220757,246 374408247 ]248 },249 "timeline_lookup": [250 [251 1,252 1900253 ],254 [255 3,256 1899257 ],258 [259 4,260 1078261 ]262 ],263 "suggested_topics": [264 {265 "fancy_title": "Autograd on a specific layer’s parameters",266 "id": 212541,267 "title": "Autograd on a specific layer's parameters",268 "slug": "autograd-on-a-specific-layers-parameters",269 "posts_count": 1,270 "reply_count": 0,271 "highest_post_number": 1,272 "image_url": null,273 "created_at": "2024-11-05T08:31:13.319Z",274 "last_posted_at": "2024-11-05T08:31:13.377Z",275 "bumped": true,276 "bumped_at": "2024-11-05T08:41:35.545Z",277 "archetype": "regular",278 "unseen": false,279 "pinned": false,280 "unpinned": null,281 "visible": true,282 "closed": false,283 "archived": false,284 "bookmarked": null,285 "liked": null,286 "tags_descriptions": {},287 "like_count": 0,288 "views": 32,289 "category_id": 7,290 "featured_link": null,291 "has_accepted_answer": false,292 "posters": [293 {294 "extras": "latest single",295 "description": "Original Poster, Most Recent Poster",296 "user": {297 "id": 72751,298 "username": "Klae_zhou",299 "name": "Klae zhou",300 "avatar_template": "/user_avatar/discuss.pytorch.org/klae_zhou/{size}/73780_2.png",301 "trust_level": 1302 }303 }304 ]305 },306 {307 "fancy_title": "Second order derivative with torch.autograd.function",308 "id": 213752,309 "title": "Second order derivative with torch.autograd.function",310 "slug": "second-order-derivative-with-torch-autograd-function",311 "posts_count": 2,312 "reply_count": 0,313 "highest_post_number": 2,314 "image_url": null,315 "created_at": "2024-12-03T16:14:45.025Z",316 "last_posted_at": "2024-12-03T22:09:09.293Z",317 "bumped": true,318 "bumped_at": "2024-12-03T22:09:09.293Z",319 "archetype": "regular",320 "unseen": false,321 "pinned": false,322 "unpinned": null,323 "visible": true,324 "closed": false,325 "archived": false,326 "bookmarked": null,327 "liked": null,328 "tags_descriptions": {},329 "like_count": 0,330 "views": 229,331 "category_id": 7,332 "featured_link": null,333 "has_accepted_answer": false,334 "posters": [335 {336 "extras": null,337 "description": "Original Poster",338 "user": {339 "id": 63171,340 "username": "bpfrd",341 "name": "bpfrd",342 "avatar_template": "/user_avatar/discuss.pytorch.org/bpfrd/{size}/57126_2.png",343 "trust_level": 1344 }345 },346 {347 "extras": "latest",348 "description": "Most Recent Poster",349 "user": {350 "id": 34294,351 "username": "AlphaBetaGamma96",352 "name": "",353 "avatar_template": "/letter_avatar_proxy/v4/letter/a/3da27b/{size}.png",354 "trust_level": 2355 }356 }357 ]358 },359 {360 "fancy_title": "Cutlass kernel causes no grad in backward",361 "id": 214074,362 "title": "Cutlass kernel causes no grad in backward",363 "slug": "cutlass-kernel-causes-no-grad-in-backward",364 "posts_count": 2,365 "reply_count": 0,366 "highest_post_number": 2,367 "image_url": null,368 "created_at": "2024-12-10T20:46:09.454Z",369 "last_posted_at": "2024-12-12T16:29:03.480Z",370 "bumped": true,371 "bumped_at": "2024-12-12T16:29:03.480Z",372 "archetype": "regular",373 "unseen": false,374 "pinned": false,375 "unpinned": null,376 "visible": true,377 "closed": false,378 "archived": false,379 "bookmarked": null,380 "liked": null,381 "tags_descriptions": {},382 "like_count": 1,383 "views": 177,384 "category_id": 7,385 "featured_link": null,386 "has_accepted_answer": true,387 "posters": [388 {389 "extras": null,390 "description": "Original Poster",391 "user": {392 "id": 81436,393 "username": "terunofuji",394 "name": "terunofuji",395 "avatar_template": "/user_avatar/discuss.pytorch.org/terunofuji/{size}/72533_2.png",396 "trust_level": 0397 }398 },399 {400 "extras": "latest",401 "description": "Most Recent Poster, Accepted Answer",402 "user": {403 "id": 41396,404 "username": "soulitzer",405 "name": "",406 "avatar_template": "/letter_avatar_proxy/v4/letter/s/839c29/{size}.png",407 "trust_level": 2408 }409 }410 ]411 },412 {413 "fancy_title": "In-place Assignment of Leaf Tensors",414 "id": 214722,415 "title": "In-place Assignment of Leaf Tensors",416 "slug": "in-place-assignment-of-leaf-tensors",417 "posts_count": 2,418 "reply_count": 0,419 "highest_post_number": 2,420 "image_url": null,421 "created_at": "2024-12-28T11:24:04.493Z",422 "last_posted_at": "2024-12-29T03:05:36.650Z",423 "bumped": true,424 "bumped_at": "2024-12-29T03:05:36.650Z",425 "archetype": "regular",426 "unseen": false,427 "pinned": false,428 "unpinned": null,429 "visible": true,430 "closed": false,431 "archived": false,432 "bookmarked": null,433 "liked": null,434 "tags_descriptions": {},435 "like_count": 0,436 "views": 157,437 "category_id": 7,438 "featured_link": null,439 "has_accepted_answer": false,440 "posters": [441 {442 "extras": null,443 "description": "Original Poster",444 "user": {445 "id": 81752,446 "username": "fastlegacycode",447 "name": "",448 "avatar_template": "/user_avatar/discuss.pytorch.org/fastlegacycode/{size}/74760_2.png",449 "trust_level": 1450 }451 },452 {453 "extras": "latest",454 "description": "Most Recent Poster",455 "user": {456 "id": 41396,457 "username": "soulitzer",458 "name": "",459 "avatar_template": "/letter_avatar_proxy/v4/letter/s/839c29/{size}.png",460 "trust_level": 2461 }462 }463 ]464 },465 {466 "fancy_title": "Grad is None confusion in the “what is torch.nn” tutorial",467 "id": 217061,468 "title": "Grad is None confusion in the \"what is torch.nn\" tutorial",469 "slug": "grad-is-none-confusion-in-the-what-is-torch-nn-tutorial",470 "posts_count": 4,471 "reply_count": 1,472 "highest_post_number": 4,473 "image_url": null,474 "created_at": "2025-02-23T21:17:11.536Z",475 "last_posted_at": "2025-02-24T13:51:12.027Z",476 "bumped": true,477 "bumped_at": "2025-02-24T13:51:12.027Z",478 "archetype": "regular",479 "unseen": false,480 "pinned": false,481 "unpinned": null,482 "visible": true,483 "closed": false,484 "archived": false,485 "bookmarked": null,486 "liked": null,487 "tags_descriptions": {},488 "like_count": 2,489 "views": 48,490 "category_id": 7,491 "featured_link": null,492 "has_accepted_answer": true,493 "posters": [494 {495 "extras": null,496 "description": "Original Poster",497 "user": {498 "id": 7372,499 "username": "hunan-rostomyan",500 "name": "Hunan Rostomyan",501 "avatar_template": "/user_avatar/discuss.pytorch.org/hunan-rostomyan/{size}/75838_2.png",502 "trust_level": 1503 }504 },505 {506 "extras": "latest",507 "description": "Most Recent Poster, Accepted Answer",508 "user": {509 "id": 3534,510 "username": "ptrblck",511 "name": "",512 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",513 "admin": true,514 "moderator": true,515 "trust_level": 2516 }517 }518 ]519 }520 ],521 "tags_descriptions": {},522 "fancy_title": "IndexError : tuple index out of range during backward pass",523 "id": 92605,524 "title": "IndexError : tuple index out of range during backward pass",525 "posts_count": 4,526 "created_at": "2020-08-12T22:00:45.868Z",527 "views": 2580,528 "reply_count": 1,529 "like_count": 1,530 "last_posted_at": "2022-11-13T02:01:58.807Z",531 "visible": true,532 "closed": false,533 "archived": false,534 "has_summary": false,535 "archetype": "regular",536 "slug": "indexerror-tuple-index-out-of-range-during-backward-pass",537 "category_id": 7,538 "word_count": 871,539 "deleted_at": null,540 "user_id": 35478,541 "featured_link": null,542 "pinned_globally": false,543 "pinned_at": null,544 "pinned_until": null,545 "image_url": null,546 "slow_mode_seconds": 0,547 "draft": null,548 "draft_key": "topic_92605",549 "draft_sequence": null,550 "unpinned": null,551 "pinned": false,552 "current_post_number": 1,553 "highest_post_number": 4,554 "deleted_by": null,555 "actions_summary": [556 {557 "id": 4,558 "count": 0,559 "hidden": false,560 "can_act": false561 },562 {563 "id": 8,564 "count": 0,565 "hidden": false,566 "can_act": false567 },568 {569 "id": 10,570 "count": 0,571 "hidden": false,572 "can_act": false573 },574 {575 "id": 7,576 "count": 0,577 "hidden": false,578 "can_act": false579 }580 ],581 "chunk_size": 20,582 "bookmarked": false,583 "topic_timer": null,584 "message_bus_last_id": 0,585 "participant_count": 3,586 "show_read_indicator": false,587 "thumbnails": null,588 "slow_mode_enabled_until": null,589 "can_vote": false,590 "vote_count": 0,591 "user_voted": false,592 "discourse_zendesk_plugin_zendesk_id": null,593 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",594 "details": {595 "can_edit": false,596 "notification_level": 1,597 "participants": [598 {599 "id": 35478,600 "username": "Yun_Lee",601 "name": "Yun Lee",602 "avatar_template": "/user_avatar/discuss.pytorch.org/yun_lee/{size}/13932_2.png",603 "post_count": 2,604 "primary_group_name": null,605 "flair_name": null,606 "flair_url": null,607 "flair_color": null,608 "flair_bg_color": null,609 "flair_group_id": null,610 "trust_level": 1611 },612 {613 "id": 211,614 "username": "albanD",615 "name": "Alban D",616 "avatar_template": "/user_avatar/discuss.pytorch.org/alband/{size}/215_2.png",617 "post_count": 1,618 "primary_group_name": null,619 "flair_name": null,620 "flair_url": null,621 "flair_color": null,622 "flair_bg_color": null,623 "flair_group_id": null,624 "admin": true,625 "moderator": true,626 "trust_level": 4627 },628 {629 "id": 60897,630 "username": "jiasongd",631 "name": "Jiasongd",632 "avatar_template": "/user_avatar/discuss.pytorch.org/jiasongd/{size}/54782_2.png",633 "post_count": 1,634 "primary_group_name": null,635 "flair_name": null,636 "flair_url": null,637 "flair_color": null,638 "flair_bg_color": null,639 "flair_group_id": null,640 "trust_level": 0641 }642 ],643 "created_by": {644 "id": 35478,645 "username": "Yun_Lee",646 "name": "Yun Lee",647 "avatar_template": "/user_avatar/discuss.pytorch.org/yun_lee/{size}/13932_2.png"648 },649 "last_poster": {650 "id": 60897,651 "username": "jiasongd",652 "name": "Jiasongd",653 "avatar_template": "/user_avatar/discuss.pytorch.org/jiasongd/{size}/54782_2.png"654 }655 },656 "bookmarks": []657 },658 {659 "post_stream": {660 "posts": [661 {662 "id": 374368,663 "name": "Fox11",664 "username": "Fox11",665 "avatar_template": "/user_avatar/discuss.pytorch.org/fox11/{size}/54756_2.png",666 "created_at": "2022-11-12T11:12:32.938Z",667 "cooked": "<p>Hello, I’m working on a deep learning project with a ViT-like backbone. When I train a model under <code>PyTorch > 1.10.1</code> (I’ve already tried <code>1.12.0</code>, <code>1.12.1</code>, <code>1.13.0</code>), my program crashed. It will show the following error:</p>\n<pre><code class=\"lang-auto\">[E ProcessGroupNCCL.cpp:737] [Rank 3] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800880 milliseconds before timing out.\n[E ProcessGroupNCCL.cpp:737] [Rank 1] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800908 milliseconds before timing out.\n[E ProcessGroupNCCL.cpp:737] [Rank 2] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800958 milliseconds before timing out.\n[E ProcessGroupNCCL.cpp:414] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. To avoid this inconsistency, we are taking the entire process down.\n[E ProcessGroupNCCL.cpp:414] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. To avoid this inconsistency, we are taking the entire process down.\nterminate called after throwing an instance of 'std::runtime_error'\n what(): [Rank 3] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800880 milliseconds before timing out.\nterminate called after throwing an instance of 'std::runtime_error'\n what(): [Rank 2] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800958 milliseconds before timing out.\n[E ProcessGroupNCCL.cpp:414] Some NCCL operations have failed or timed out. Due to the asynchronous nature of CUDA kernels, subsequent GPU operations might run on corrupted/incomplete data. To avoid this inconsistency, we are taking the entire process down.\nterminate called after throwing an instance of 'std::runtime_error'\n what(): [Rank 1] Watchdog caught collective operation timeout: WorkNCCL(SeqNum=63841, OpType=BROADCAST, Timeout(ms)=1800000) ran for 1800908 milliseconds before timing out.\nWARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1000892 closing signal SIGTERM\nERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 1 (pid: 1000893) of binary: /opt/conda/envs/lwh3/bin/python\nTraceback (most recent call last):\n File \"/opt/conda/envs/lwh3/lib/python3.8/runpy.py\", line 194, in _run_module_as_main\n return _run_code(code, main_globals, None,\n File \"/opt/conda/envs/lwh3/lib/python3.8/runpy.py\", line 87, in _run_code\n exec(code, run_globals)\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/launch.py\", line 193, in <module>\n main()\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/launch.py\", line 189, in main\n launch(args)\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/launch.py\", line 174, in launch\n run(args)\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/run.py\", line 752, in run\n elastic_launch(\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/launcher/api.py\", line 131, in __call__\n return launch_agent(self._config, self._entrypoint, list(args))\n File \"/opt/conda/envs/lwh3/lib/python3.8/site-packages/torch/distributed/launcher/api.py\", line 245, in launch_agent\n raise ChildFailedError(\ntorch.distributed.elastic.multiprocessing.errors.ChildFailedError: \n========================================================\nlib/train/run_training.py FAILED\n--------------------------------------------------------\nFailures:\n[1]:\n time : 2022-11-12_10:44:17\n host : ed85ab297bc3\n rank : 2 (local_rank: 2)\n exitcode : -6 (pid: 1000894)\n error_file: <N/A>\n traceback : Signal 6 (SIGABRT) received by PID 1000894\n[2]:\n time : 2022-11-12_10:44:17\n host : ed85ab297bc3\n rank : 3 (local_rank: 3)\n exitcode : -6 (pid: 1000895)\n error_file: <N/A>\n traceback : Signal 6 (SIGABRT) received by PID 1000895\n--------------------------------------------------------\nRoot Cause (first observed failure):\n[0]:\n time : 2022-11-12_10:44:17\n host : ed85ab297bc3\n rank : 1 (local_rank: 1)\n exitcode : -6 (pid: 1000893)\n error_file: <N/A>\n traceback : Signal 6 (SIGABRT) received by PID 1000893\n========================================================\n</code></pre>\n<p>When I use <code>PyTorch 1.10.1</code>, everything went well, however I need to use some funtions which only exist in <code>PyTorch >= 1.12.0</code>. This problem really confused me, I’m wondering if anyone could help me with this？</p>",668 "post_number": 1,669 "post_type": 1,670 "posts_count": 2,671 "updated_at": "2022-11-12T12:23:22.493Z",672 "reply_count": 0,673 "reply_to_post_number": null,674 "quote_count": 0,675 "incoming_link_count": 553,676 "reads": 19,677 "readers_count": 18,678 "score": 2768.8,679 "yours": false,680 "topic_id": 165792,681 "topic_slug": "my-python-program-crashed-with-pytorch-1-10-1",682 "display_username": "Fox11",683 "primary_group_name": null,684 "flair_name": null,685 "flair_url": null,686 "flair_bg_color": null,687 "flair_color": null,688 "flair_group_id": null,689 "badges_granted": [],690 "version": 2,691 "can_edit": false,692 "can_delete": false,693 "can_recover": false,694 "can_see_hidden_post": false,695 "can_wiki": false,696 "read": true,697 "user_title": null,698 "bookmarked": false,699 "actions_summary": [],700 "moderator": false,701 "admin": false,702 "staff": false,703 "user_id": 60868,704 "hidden": false,705 "trust_level": 1,706 "deleted_at": null,707 "user_deleted": false,708 "edit_reason": null,709 "can_view_edit_history": true,710 "wiki": false,711 "post_url": "/t/my-python-program-crashed-with-pytorch-1-10-1/165792/1",712 "can_accept_answer": false,713 "can_unaccept_answer": false,714 "accepted_answer": false,715 "topic_accepted_answer": null,716 "can_vote": false717 },718 {719 "id": 374402,720 "name": "",721 "username": "ptrblck",722 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",723 "created_at": "2022-11-12T20:41:08.407Z",724 "cooked": "<p>Rerun your code with <code>export NCCL_DEBUG=INFO</code> and check which errors or warnings NCCL raises before crashing.</p>",725 "post_number": 2,726 "post_type": 1,727 "posts_count": 2,728 "updated_at": "2022-11-12T20:41:08.407Z",729 "reply_count": 0,730 "reply_to_post_number": null,731 "quote_count": 0,732 "incoming_link_count": 3,733 "reads": 19,734 "readers_count": 18,735 "score": 18.8,736 "yours": false,737 "topic_id": 165792,738 "topic_slug": "my-python-program-crashed-with-pytorch-1-10-1",739 "display_username": "",740 "primary_group_name": null,741 "flair_name": null,742 "flair_url": null,743 "flair_bg_color": null,744 "flair_color": null,745 "flair_group_id": null,746 "badges_granted": [],747 "version": 1,748 "can_edit": false,749 "can_delete": false,750 "can_recover": false,751 "can_see_hidden_post": false,752 "can_wiki": false,753 "read": true,754 "user_title": "",755 "bookmarked": false,756 "actions_summary": [],757 "moderator": true,758 "admin": true,759 "staff": true,760 "user_id": 3534,761 "hidden": false,762 "trust_level": 2,763 "deleted_at": null,764 "user_deleted": false,765 "edit_reason": null,766 "can_view_edit_history": true,767 "wiki": false,768 "post_url": "/t/my-python-program-crashed-with-pytorch-1-10-1/165792/2",769 "can_accept_answer": false,770 "can_unaccept_answer": false,771 "accepted_answer": false,772 "topic_accepted_answer": null773 }774 ],775 "stream": [776 374368,777 374402778 ]779 },780 "timeline_lookup": [781 [782 1,783 1078784 ]785 ],786 "suggested_topics": [787 {788 "fancy_title": "Multiple Models Performance Degrades",789 "id": 217997,790 "title": "Multiple Models Performance Degrades",791 "slug": "multiple-models-performance-degrades",792 "posts_count": 1,793 "reply_count": 0,794 "highest_post_number": 1,795 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/1/c/1c795da9ca2096fc66e9c4b0b865a7995bda649b_2_1024x315.png",796 "created_at": "2025-03-18T18:03:18.722Z",797 "last_posted_at": "2025-03-18T18:03:18.772Z",798 "bumped": true,799 "bumped_at": "2025-03-18T18:03:18.772Z",800 "archetype": "regular",801 "unseen": false,802 "pinned": false,803 "unpinned": null,804 "visible": true,805 "closed": false,806 "archived": false,807 "bookmarked": null,808 "liked": null,809 "tags_descriptions": {},810 "like_count": 0,811 "views": 28,812 "category_id": 1,813 "featured_link": null,814 "has_accepted_answer": false,815 "posters": [816 {817 "extras": "latest single",818 "description": "Original Poster, Most Recent Poster",819 "user": {820 "id": 81841,821 "username": "goofydoge",822 "name": null,823 "avatar_template": "/letter_avatar_proxy/v4/letter/g/8c91f0/{size}.png",824 "trust_level": 1825 }826 }827 ]828 },829 {830 "fancy_title": "Inverse function of `logaddexp`?",831 "id": 215936,832 "title": "Inverse function of `logaddexp`?",833 "slug": "inverse-function-of-logaddexp",834 "posts_count": 4,835 "reply_count": 0,836 "highest_post_number": 4,837 "image_url": null,838 "created_at": "2025-01-27T14:27:15.031Z",839 "last_posted_at": "2025-01-27T15:58:48.845Z",840 "bumped": true,841 "bumped_at": "2025-01-27T16:21:05.911Z",842 "archetype": "regular",843 "unseen": false,844 "pinned": false,845 "unpinned": null,846 "visible": true,847 "closed": false,848 "archived": false,849 "bookmarked": null,850 "liked": null,851 "tags_descriptions": {},852 "like_count": 0,853 "views": 207,854 "category_id": 1,855 "featured_link": null,856 "has_accepted_answer": false,857 "posters": [858 {859 "extras": "latest single",860 "description": "Original Poster, Most Recent Poster",861 "user": {862 "id": 60714,863 "username": "jakelevi1996",864 "name": "Jake Levi",865 "avatar_template": "/user_avatar/discuss.pytorch.org/jakelevi1996/{size}/75757_2.png",866 "trust_level": 2867 }868 }869 ]870 },871 {872 "fancy_title": "RuntimeError: Input tensors need to be on the same GPU",873 "id": 217286,874 "title": "RuntimeError: Input tensors need to be on the same GPU",875 "slug": "runtimeerror-input-tensors-need-to-be-on-the-same-gpu",876 "posts_count": 1,877 "reply_count": 0,878 "highest_post_number": 1,879 "image_url": null,880 "created_at": "2025-02-28T12:44:11.593Z",881 "last_posted_at": "2025-02-28T12:44:11.634Z",882 "bumped": true,883 "bumped_at": "2025-02-28T12:44:11.634Z",884 "archetype": "regular",885 "unseen": false,886 "pinned": false,887 "unpinned": null,888 "visible": true,889 "closed": false,890 "archived": false,891 "bookmarked": null,892 "liked": null,893 "tags_descriptions": {},894 "like_count": 0,895 "views": 57,896 "category_id": 1,897 "featured_link": null,898 "has_accepted_answer": false,899 "posters": [900 {901 "extras": "latest single",902 "description": "Original Poster, Most Recent Poster",903 "user": {904 "id": 73637,905 "username": "chrathans",906 "name": "Kris Tosh",907 "avatar_template": "/user_avatar/discuss.pytorch.org/chrathans/{size}/67980_2.png",908 "trust_level": 1909 }910 }911 ]912 },913 {914 "fancy_title": "Persistent error in loss.backward() during joint-training of two models using two different but dependent loss functions",915 "id": 212593,916 "title": "Persistent error in loss.backward() during joint-training of two models using two different but dependent loss functions",917 "slug": "persistent-error-in-loss-backward-during-joint-training-of-two-models-using-two-different-but-dependent-loss-functions",918 "posts_count": 1,919 "reply_count": 0,920 "highest_post_number": 1,921 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/3/d/3db4085e6be45f34ad9d0a6ff92d9f7a375fa71d_2_1024x664.png",922 "created_at": "2024-11-06T05:47:29.578Z",923 "last_posted_at": "2024-11-06T05:47:29.688Z",924 "bumped": true,925 "bumped_at": "2024-11-06T05:47:29.688Z",926 "archetype": "regular",927 "unseen": false,928 "pinned": false,929 "unpinned": null,930 "visible": true,931 "closed": false,932 "archived": false,933 "bookmarked": null,934 "liked": null,935 "tags_descriptions": {},936 "like_count": 0,937 "views": 10,938 "category_id": 1,939 "featured_link": null,940 "has_accepted_answer": false,941 "posters": [942 {943 "extras": "latest single",944 "description": "Original Poster, Most Recent Poster",945 "user": {946 "id": 80719,947 "username": "Goutam",948 "name": "Goutam",949 "avatar_template": "/user_avatar/discuss.pytorch.org/goutam/{size}/73797_2.png",950 "trust_level": 0951 }952 }953 ]954 },955 {956 "fancy_title": "Best way to compute C = A.inv() @ B @ A.inv().T? Linear algebra",957 "id": 220883,958 "title": "Best way to compute C = A.inv() @ B @ A.inv().T? Linear algebra",959 "slug": "best-way-to-compute-c-a-inv-b-a-inv-t-linear-algebra",960 "posts_count": 2,961 "reply_count": 0,962 "highest_post_number": 2,963 "image_url": null,964 "created_at": "2025-06-17T19:59:14.950Z",965 "last_posted_at": "2025-06-19T04:56:08.638Z",966 "bumped": true,967 "bumped_at": "2025-06-19T04:56:08.638Z",968 "archetype": "regular",969 "unseen": false,970 "pinned": false,971 "unpinned": null,972 "visible": true,973 "closed": false,974 "archived": false,975 "bookmarked": null,976 "liked": null,977 "tags_descriptions": {},978 "like_count": 1,979 "views": 43,980 "category_id": 1,981 "featured_link": null,982 "has_accepted_answer": true,983 "posters": [984 {985 "extras": null,986 "description": "Original Poster",987 "user": {988 "id": 67552,989 "username": "dherrera1911",990 "name": "Daniel Herrera",991 "avatar_template": "/user_avatar/discuss.pytorch.org/dherrera1911/{size}/61905_2.png",992 "trust_level": 1993 }994 },995 {996 "extras": "latest",997 "description": "Most Recent Poster, Accepted Answer",998 "user": {999 "id": 18088,1000 "username": "KFrank",1001 "name": "K. Frank",1002 "avatar_template": "/letter_avatar_proxy/v4/letter/k/ecb155/{size}.png",1003 "trust_level": 21004 }1005 }1006 ]1007 }1008 ],1009 "tags_descriptions": {},1010 "fancy_title": "My python program crashed with PyTorch >1.10.1",1011 "id": 165792,1012 "title": "My python program crashed with PyTorch >1.10.1",1013 "posts_count": 2,1014 "created_at": "2022-11-12T11:12:32.841Z",1015 "views": 1070,1016 "reply_count": 0,1017 "like_count": 0,1018 "last_posted_at": "2022-11-12T20:41:08.407Z",1019 "visible": true,1020 "closed": false,1021 "archived": false,1022 "has_summary": false,1023 "archetype": "regular",1024 "slug": "my-python-program-crashed-with-pytorch-1-10-1",1025 "category_id": 1,1026 "word_count": 698,1027 "deleted_at": null,1028 "user_id": 60868,1029 "featured_link": null,1030 "pinned_globally": false,1031 "pinned_at": null,1032 "pinned_until": null,1033 "image_url": null,1034 "slow_mode_seconds": 0,1035 "draft": null,1036 "draft_key": "topic_165792",1037 "draft_sequence": null,1038 "unpinned": null,1039 "pinned": false,1040 "current_post_number": 1,1041 "highest_post_number": 2,1042 "deleted_by": null,1043 "actions_summary": [1044 {1045 "id": 4,1046 "count": 0,1047 "hidden": false,1048 "can_act": false1049 },1050 {1051 "id": 8,1052 "count": 0,1053 "hidden": false,1054 "can_act": false1055 },1056 {1057 "id": 10,1058 "count": 0,1059 "hidden": false,1060 "can_act": false1061 },1062 {1063 "id": 7,1064 "count": 0,1065 "hidden": false,1066 "can_act": false1067 }1068 ],1069 "chunk_size": 20,1070 "bookmarked": false,1071 "topic_timer": null,1072 "message_bus_last_id": 0,1073 "participant_count": 2,1074 "show_read_indicator": false,1075 "thumbnails": null,1076 "slow_mode_enabled_until": null,1077 "can_vote": false,1078 "vote_count": 0,1079 "user_voted": false,1080 "discourse_zendesk_plugin_zendesk_id": null,1081 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",1082 "details": {1083 "can_edit": false,1084 "notification_level": 1,1085 "participants": [1086 {1087 "id": 3534,1088 "username": "ptrblck",1089 "name": "",1090 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1091 "post_count": 1,1092 "primary_group_name": null,1093 "flair_name": null,1094 "flair_url": null,1095 "flair_color": null,1096 "flair_bg_color": null,1097 "flair_group_id": null,1098 "admin": true,1099 "moderator": true,1100 "trust_level": 21101 },1102 {1103 "id": 60868,1104 "username": "Fox11",1105 "name": "Fox11",1106 "avatar_template": "/user_avatar/discuss.pytorch.org/fox11/{size}/54756_2.png",1107 "post_count": 1,1108 "primary_group_name": null,1109 "flair_name": null,1110 "flair_url": null,1111 "flair_color": null,1112 "flair_bg_color": null,1113 "flair_group_id": null,1114 "trust_level": 11115 }1116 ],1117 "created_by": {1118 "id": 60868,1119 "username": "Fox11",1120 "name": "Fox11",1121 "avatar_template": "/user_avatar/discuss.pytorch.org/fox11/{size}/54756_2.png"1122 },1123 "last_poster": {1124 "id": 3534,1125 "username": "ptrblck",1126 "name": "",1127 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"1128 }1129 },1130 "bookmarks": []1131 },1132 {1133 "post_stream": {1134 "posts": [1135 {1136 "id": 374379,1137 "name": "mrleo194",1138 "username": "mrleo194",1139 "avatar_template": "/user_avatar/discuss.pytorch.org/mrleo194/{size}/54775_2.png",1140 "created_at": "2022-11-12T14:33:21.556Z",1141 "cooked": "<p>I want to build a dataset for segmentation for EndoCV 2020 dataset. In this dataset, for example, <code>image = abc.jpg</code>, with 2 classes in this image. The corresponding masks are <code>mask_1 = abc_class1.tif</code> and ‘mask_2 = abc_class2.tif’. How can I build a dataset that PyTorch can train?</p>",1142 "post_number": 1,1143 "post_type": 1,1144 "posts_count": 2,1145 "updated_at": "2022-11-12T14:33:21.556Z",1146 "reply_count": 0,1147 "reply_to_post_number": null,1148 "quote_count": 0,1149 "incoming_link_count": 22,1150 "reads": 6,1151 "readers_count": 5,1152 "score": 111.2,1153 "yours": false,1154 "topic_id": 165798,1155 "topic_slug": "build-dataset-with-multiple-class-masks-for-endocv-2020",1156 "display_username": "mrleo194",1157 "primary_group_name": null,1158 "flair_name": null,1159 "flair_url": null,1160 "flair_bg_color": null,1161 "flair_color": null,1162 "flair_group_id": null,1163 "badges_granted": [],1164 "version": 1,1165 "can_edit": false,1166 "can_delete": false,1167 "can_recover": false,1168 "can_see_hidden_post": false,1169 "can_wiki": false,1170 "read": true,1171 "user_title": null,1172 "bookmarked": false,1173 "actions_summary": [],1174 "moderator": false,1175 "admin": false,1176 "staff": false,1177 "user_id": 60894,1178 "hidden": false,1179 "trust_level": 1,1180 "deleted_at": null,1181 "user_deleted": false,1182 "edit_reason": null,1183 "can_view_edit_history": true,1184 "wiki": false,1185 "post_url": "/t/build-dataset-with-multiple-class-masks-for-endocv-2020/165798/1",1186 "can_accept_answer": false,1187 "can_unaccept_answer": false,1188 "accepted_answer": false,1189 "topic_accepted_answer": null,1190 "can_vote": false1191 },1192 {1193 "id": 374401,1194 "name": "",1195 "username": "ptrblck",1196 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1197 "created_at": "2022-11-12T20:36:59.421Z",1198 "cooked": "<p><a href=\"https://pytorch.org/tutorials/beginner/data_loading_tutorial.html\">This tutorial</a> might be a good starter which explains how to write a custom <code>Dataset</code>.</p>",1199 "post_number": 2,1200 "post_type": 1,