Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 411315,7 "name": "Sebastian Hurubaru",8 "username": "SebastianHurubaru",9 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",10 "created_at": "2023-07-25T09:14:10.551Z",11 "cooked": "<p>Hello,</p>\n<p>we reported this performance issue between the two PyTorch version as <a href=\"https://github.com/pytorch/pytorch/issues/105837\" rel=\"noopener nofollow ugc\">#105837</a> GitHub issue.</p>\n<p>As a new user cannot seem to add more links or media, so all the info should be found in the GitHub issue.</p>\n<p>Could someone please take a look?</p>\n<p>Thank you in advance!</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 17,15 "updated_at": "2023-07-25T09:14:10.551Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 1342,20 "reads": 44,21 "readers_count": 43,22 "score": 6708.8,23 "yours": false,24 "topic_id": 184989,25 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",26 "display_username": "Sebastian Hurubaru",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 1,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "link_counts": [41 {42 "url": "https://github.com/pytorch/pytorch/issues/105837",43 "internal": false,44 "reflection": false,45 "title": "Big accuracy differences between PyTorch 1.13.1 and 2.0.1! · Issue #105837 · pytorch/pytorch · GitHub",46 "clicks": 3847 }48 ],49 "read": true,50 "user_title": null,51 "bookmarked": false,52 "actions_summary": [],53 "moderator": false,54 "admin": false,55 "staff": false,56 "user_id": 68141,57 "hidden": false,58 "trust_level": 1,59 "deleted_at": null,60 "user_deleted": false,61 "edit_reason": null,62 "can_view_edit_history": true,63 "wiki": false,64 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/1",65 "can_accept_answer": false,66 "can_unaccept_answer": false,67 "accepted_answer": false,68 "topic_accepted_answer": true,69 "can_vote": false70 },71 {72 "id": 411317,73 "name": "",74 "username": "ptrblck",75 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",76 "created_at": "2023-07-25T10:00:26.051Z",77 "cooked": "<p>Did you check if any parameter updates are performed or if the model is static?</p>",78 "post_number": 2,79 "post_type": 1,80 "posts_count": 17,81 "updated_at": "2023-07-25T10:00:26.051Z",82 "reply_count": 0,83 "reply_to_post_number": null,84 "quote_count": 0,85 "incoming_link_count": 2,86 "reads": 42,87 "readers_count": 41,88 "score": 18.4,89 "yours": false,90 "topic_id": 184989,91 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",92 "display_username": "",93 "primary_group_name": null,94 "flair_name": null,95 "flair_url": null,96 "flair_bg_color": null,97 "flair_color": null,98 "flair_group_id": null,99 "badges_granted": [],100 "version": 1,101 "can_edit": false,102 "can_delete": false,103 "can_recover": false,104 "can_see_hidden_post": false,105 "can_wiki": false,106 "read": true,107 "user_title": "",108 "bookmarked": false,109 "actions_summary": [],110 "moderator": true,111 "admin": true,112 "staff": true,113 "user_id": 3534,114 "hidden": false,115 "trust_level": 2,116 "deleted_at": null,117 "user_deleted": false,118 "edit_reason": null,119 "can_view_edit_history": true,120 "wiki": false,121 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/2",122 "can_accept_answer": false,123 "can_unaccept_answer": false,124 "accepted_answer": false,125 "topic_accepted_answer": true126 },127 {128 "id": 411385,129 "name": "Sebastian Hurubaru",130 "username": "SebastianHurubaru",131 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",132 "created_at": "2023-07-25T18:47:38.923Z",133 "cooked": "<p>Thanks <a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> for your answer! Checked with the following code whether the parameters get updated and they seem to be not, as the original plots submitted also show that no learning is happening.</p>\n<pre><code class=\"lang-auto\">optim.zero_grad()\n\nparams_before = list(model.parameters())\n\nloss.backward()\noptim.step()\n\nparams_after = list(model.parameters())\n\nall_params_equal = True\nfor p_before, p_after in zip(params_before, params_after):\n all_params_equal = all_params_equal and torch.equal(p_before.data, p_after.data)\n\nprint(f\"Are all parameters equal? {all_params_equal}\")\n</code></pre>\n<p>Not sure what you mean by checking whether the model is static.</p>\n<p>Thanks again!</p>",134 "post_number": 3,135 "post_type": 1,136 "posts_count": 17,137 "updated_at": "2023-07-25T18:48:02.617Z",138 "reply_count": 1,139 "reply_to_post_number": null,140 "quote_count": 0,141 "incoming_link_count": 4,142 "reads": 36,143 "readers_count": 35,144 "score": 32.2,145 "yours": false,146 "topic_id": 184989,147 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",148 "display_username": "Sebastian Hurubaru",149 "primary_group_name": null,150 "flair_name": null,151 "flair_url": null,152 "flair_bg_color": null,153 "flair_color": null,154 "flair_group_id": null,155 "badges_granted": [],156 "version": 1,157 "can_edit": false,158 "can_delete": false,159 "can_recover": false,160 "can_see_hidden_post": false,161 "can_wiki": false,162 "read": true,163 "user_title": null,164 "bookmarked": false,165 "actions_summary": [],166 "moderator": false,167 "admin": false,168 "staff": false,169 "user_id": 68141,170 "hidden": false,171 "trust_level": 1,172 "deleted_at": null,173 "user_deleted": false,174 "edit_reason": null,175 "can_view_edit_history": true,176 "wiki": false,177 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/3",178 "can_accept_answer": false,179 "can_unaccept_answer": false,180 "accepted_answer": false,181 "topic_accepted_answer": true182 },183 {184 "id": 411396,185 "name": "",186 "username": "ptrblck",187 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",188 "created_at": "2023-07-25T20:44:29.807Z",189 "cooked": "<p>Thanks for the check! Could you check the <code>.grad</code> attributes of all parameters before and after the first <code>backward()</code> call next?<br>\nThey should be set to <code>None</code> before and should show a valid tensor afterwards.<br>\nIf both <code>print</code> statements are showing <code>None</code> gradients, the computation graph seems to be detached and we would need to check why that’s the case.</p>",190 "post_number": 4,191 "post_type": 1,192 "posts_count": 17,193 "updated_at": "2023-07-25T20:44:29.807Z",194 "reply_count": 1,195 "reply_to_post_number": 3,196 "quote_count": 0,197 "incoming_link_count": 0,198 "reads": 30,199 "readers_count": 29,200 "score": 11.0,201 "yours": false,202 "topic_id": 184989,203 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",204 "display_username": "",205 "primary_group_name": null,206 "flair_name": null,207 "flair_url": null,208 "flair_bg_color": null,209 "flair_color": null,210 "flair_group_id": null,211 "badges_granted": [],212 "version": 1,213 "can_edit": false,214 "can_delete": false,215 "can_recover": false,216 "can_see_hidden_post": false,217 "can_wiki": false,218 "read": true,219 "user_title": "",220 "reply_to_user": {221 "id": 68141,222 "username": "SebastianHurubaru",223 "name": "Sebastian Hurubaru",224 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"225 },226 "bookmarked": false,227 "actions_summary": [],228 "moderator": true,229 "admin": true,230 "staff": true,231 "user_id": 3534,232 "hidden": false,233 "trust_level": 2,234 "deleted_at": null,235 "user_deleted": false,236 "edit_reason": null,237 "can_view_edit_history": true,238 "wiki": false,239 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/4",240 "can_accept_answer": false,241 "can_unaccept_answer": false,242 "accepted_answer": false,243 "topic_accepted_answer": true244 },245 {246 "id": 411457,247 "name": "Sebastian Hurubaru",248 "username": "SebastianHurubaru",249 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",250 "created_at": "2023-07-26T07:48:18.084Z",251 "cooked": "<aside class=\"quote no-group\" data-username=\"ptrblck\" data-post=\"4\" data-topic=\"184989\">\n<div class=\"title\">\n<div class=\"quote-controls\"></div>\n<img loading=\"lazy\" alt=\"\" width=\"24\" height=\"24\" src=\"https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ptrblck/48/1823_2.png\" class=\"avatar\"> ptrblck:</div>\n<blockquote>\n<p>.grad</p>\n</blockquote>\n</aside>\n<p>Thanks again <a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> . Modified the training loop as below:</p>\n<pre><code class=\"lang-auto\">optim.zero_grad()\n\nparams_before = list([param.data.cpu().detach().numpy() for param in model.parameters()])\n\nfor p_before in model.parameters():\n print(f\"Grad: before - {p_before.grad}\")\n\nloss.backward()\n\nfor p_after in model.parameters():\n print(f\"Grad: after - {p_after.grad}\")\n\noptim.step()\n\nparams_after = list([param.data.cpu().detach().numpy() for param in model.parameters()])\n\nall_params_equal = True\nfor p_before, p_after in zip(params_before, params_after):\n all_params_equal = all_params_equal and np.array_equal(p_before, p_after)\n\nprint(f\"Are all parameters equal? {all_params_equal}\")\n</code></pre>\n<p>Previously I think the parameters shared the same reference, and were marked as equal. Detached them and stored them as numpy arrays to make the comparison and there seem to be small changes in the parameters, i.e. they seem to get updated. Sorry for this!</p>\n<p>The gradients seem to be None first and then after doing first the backprop they seem to get updated to tensors. Here is the output:</p>\n<pre><code class=\"lang-auto\">Grad: before - None\nGrad: before - None\nGrad: before - None\nGrad: before - None\nGrad: before - None\nGrad: before - None\nGrad: before - None\nGrad: before - None\nGrad: after - tensor([[ 1.4627e-03, 1.4627e-03, 1.4627e-03, ..., 1.4627e-03,\n 2.0679e-03, 1.3948e-03],\n [-1.3086e-04, -1.3086e-04, -1.3086e-04, ..., -1.3086e-04,\n -3.8431e-04, -5.9046e-06],\n [-6.0961e-03, -6.0961e-03, -6.0961e-03, ..., -6.0961e-03,\n -9.0613e-04, -4.6752e-03],\n ...,\n [-1.9681e-02, -1.9681e-02, -1.9681e-02, ..., -1.9681e-02,\n 1.2326e-02, 1.6755e-03],\n [ 1.2912e-03, 1.2912e-03, 1.2912e-03, ..., 1.2912e-03,\n -1.2034e-03, 2.9203e-03],\n [-1.9582e-02, -1.9582e-02, -1.9582e-02, ..., -1.9582e-02,\n 4.8703e-04, -2.1845e-03]], device='mps:0')\nGrad: after - tensor([[-1.8982e-03, -6.2929e-04, -1.5961e-03, ..., -8.6900e-04,\n -1.0032e-03, 3.9843e-04],\n [ 1.8086e-04, 2.5083e-04, -1.6416e-04, ..., 9.0401e-05,\n -1.1745e-04, 1.5602e-04],\n [-1.5279e-03, -5.0708e-03, 2.8829e-03, ..., -2.1105e-03,\n 3.0370e-03, -4.7238e-03],\n ...,\n [-2.6152e-02, -3.1857e-02, 1.3831e-02, ..., -1.3835e-02,\n 1.1094e-02, -1.8385e-02],\n [ 1.2020e-03, 2.4224e-03, 6.7051e-04, ..., 1.7220e-03,\n -5.1741e-04, 2.1676e-03],\n [-1.4955e-02, -1.9936e-02, 8.3015e-03, ..., -8.8695e-03,\n 7.3607e-03, -1.2555e-02]], device='mps:0')\nGrad: after - tensor([ 1.4685e-03, -1.3138e-04, -6.1201e-03, -4.5789e-02, -7.9349e-03,\n -1.6080e-03, 1.7505e-02, 2.7263e-02, 5.0776e-04, 3.1822e-03,\n 2.4568e-02, -1.1433e-02, 3.9419e-03, 1.5574e-02, 6.2263e-04,\n -1.2129e-02, -5.4287e-02, -1.0878e-04, -1.1254e-02, 1.0963e-02,\n 3.1155e-02, -5.8268e-03, 2.4303e-02, 4.3852e-03, -1.9106e-02,\n -1.5162e-02, -6.5663e-02, 8.1562e-02, 2.1123e-02, -3.0827e-03,\n 1.7691e-02, 1.1239e-03, -3.1183e-03, -1.0129e-02, 1.1770e-03,\n -9.5528e-04, -7.8951e-04, 5.4114e-02, 5.2791e-05, 2.3061e-05,\n 1.3466e-02, 1.1075e-03, 6.7796e-04, 1.1105e-01, -1.3704e-02,\n 1.6770e-02, -2.0077e-03, 3.3308e-05, 6.1390e-02, -1.2727e-02,\n -5.9857e-02, 7.5183e-03, -2.3664e-02, -1.4469e-02, 3.3819e-02,\n 3.1799e-02, 4.5254e-03, -2.6284e-03, 6.6589e-04, -1.3513e-03,\n -7.4101e-03, -2.9122e-03, 2.5119e-03, -5.5881e-03, 3.6866e-03,\n 4.8116e-02, 4.3276e-03, -1.4491e-02, -6.1160e-02, 1.9650e-03,\n -2.2383e-02, -1.4282e-03, 6.4808e-04, 2.0432e-02, -6.1903e-02,\n 4.2811e-02, 2.0047e-02, -4.6595e-04, 4.8694e-04, 4.7563e-02,\n 6.9717e-02, -3.4239e-02, -3.8569e-02, -2.3586e-04, -1.8914e-02,\n -3.5095e-03, -2.3049e-03, -3.1967e-03, 1.2116e-02, -3.9122e-03,\n -2.7179e-03, -2.0984e-02, 1.8846e-02, -1.6740e-02, -9.4212e-03,\n -5.8745e-04, -2.2158e-02, 1.4861e-02, -3.1859e-02, -3.9706e-03,\n -4.2073e-02, -3.3877e-02, -3.7556e-02, 5.8694e-04, 2.3499e-02,\n 6.8007e-02, -2.4977e-03, -2.4801e-02, -2.3341e-02, -7.4005e-03,\n 7.1274e-04, -5.4910e-03, 1.8000e-03, -2.8609e-03, -7.8491e-03,\n -1.7710e-02, 5.1535e-03, -6.2510e-02, -1.0905e-03, -2.0986e-02,\n -2.9979e-02, -1.9756e-03, 1.6864e-02, 5.0562e-03, 1.0459e-03,\n -1.9758e-02, 1.2963e-03, -1.9659e-02], device='mps:0')\nGrad: after - tensor([ 1.4685e-03, -1.3138e-04, -6.1201e-03, -4.5789e-02, -7.9349e-03,\n -1.6080e-03, 1.7505e-02, 2.7263e-02, 5.0776e-04, 3.1822e-03,\n 2.4568e-02, -1.1433e-02, 3.9419e-03, 1.5574e-02, 6.2263e-04,\n -1.2129e-02, -5.4287e-02, -1.0878e-04, -1.1254e-02, 1.0963e-02,\n 3.1155e-02, -5.8268e-03, 2.4303e-02, 4.3852e-03, -1.9106e-02,\n -1.5162e-02, -6.5663e-02, 8.1562e-02, 2.1123e-02, -3.0827e-03,\n 1.7691e-02, 1.1239e-03, -3.1183e-03, -1.0129e-02, 1.1770e-03,\n -9.5528e-04, -7.8951e-04, 5.4114e-02, 5.2791e-05, 2.3061e-05,\n 1.3466e-02, 1.1075e-03, 6.7796e-04, 1.1105e-01, -1.3704e-02,\n 1.6770e-02, -2.0077e-03, 3.3308e-05, 6.1390e-02, -1.2727e-02,\n -5.9857e-02, 7.5183e-03, -2.3664e-02, -1.4469e-02, 3.3819e-02,\n 3.1799e-02, 4.5254e-03, -2.6284e-03, 6.6589e-04, -1.3513e-03,\n -7.4101e-03, -2.9122e-03, 2.5119e-03, -5.5881e-03, 3.6866e-03,\n 4.8116e-02, 4.3276e-03, -1.4491e-02, -6.1160e-02, 1.9650e-03,\n -2.2383e-02, -1.4282e-03, 6.4808e-04, 2.0432e-02, -6.1903e-02,\n 4.2811e-02, 2.0047e-02, -4.6595e-04, 4.8694e-04, 4.7563e-02,\n 6.9717e-02, -3.4239e-02, -3.8569e-02, -2.3586e-04, -1.8914e-02,\n -3.5095e-03, -2.3049e-03, -3.1967e-03, 1.2116e-02, -3.9122e-03,\n -2.7179e-03, -2.0984e-02, 1.8846e-02, -1.6740e-02, -9.4212e-03,\n -5.8745e-04, -2.2158e-02, 1.4861e-02, -3.1859e-02, -3.9706e-03,\n -4.2073e-02, -3.3877e-02, -3.7556e-02, 5.8694e-04, 2.3499e-02,\n 6.8007e-02, -2.4977e-03, -2.4801e-02, -2.3341e-02, -7.4005e-03,\n 7.1274e-04, -5.4910e-03, 1.8000e-03, -2.8609e-03, -7.8491e-03,\n -1.7710e-02, 5.1535e-03, -6.2510e-02, -1.0905e-03, -2.0986e-02,\n -2.9979e-02, -1.9756e-03, 1.6864e-02, 5.0562e-03, 1.0459e-03,\n -1.9758e-02, 1.2963e-03, -1.9659e-02], device='mps:0')\nGrad: after - tensor([[ 2.4017e-01, 3.4895e-01, -2.4161e-01, 1.6617e-01, 1.4000e-02,\n -2.8061e-01, -2.4008e-01, 1.9611e-01, 2.1064e-01, 3.3362e-01,\n -2.3464e-01, -3.1525e-01, 3.0090e-01, -1.4328e-01, 3.2811e-01,\n -2.5250e-01, 1.2876e-03, -3.4962e-01, -3.2906e-01, 2.3347e-01,\n -2.5695e-01, -3.1041e-01, 9.6729e-02, -3.3356e-01, -1.7707e-01,\n 3.0241e-01, 3.8306e-02, -1.7197e-01, 7.4984e-02, 1.7992e-01,\n 3.1127e-01, 3.4695e-01, -3.4585e-01, -2.8037e-01, 3.4493e-01,\n -3.1164e-01, -3.0445e-01, 3.9136e-02, 3.4769e-01, -3.5068e-01,\n -2.8867e-01, 3.0238e-01, 3.4511e-01, -6.2755e-02, -2.9789e-01,\n -2.5160e-01, -3.4609e-01, 3.5162e-01, -1.4761e-01, -2.9846e-01,\n -2.2022e-01, 1.1042e-01, 2.8762e-01, 3.0170e-01, -2.2555e-01,\n 2.2018e-01, -3.3848e-01, 3.5028e-01, -3.4993e-01, -2.3353e-01,\n -3.2610e-01, -3.3896e-01, 3.4588e-01, 3.3694e-01, 3.4409e-01,\n 2.5323e-01, 3.2930e-01, 2.4329e-01, 2.3358e-01, -7.9981e-02,\n 2.9703e-01, 3.4929e-01, 3.5171e-01, -3.1841e-01, 1.9108e-01,\n -2.4899e-01, 1.6254e-01, 3.5132e-01, -3.4985e-01, 1.5251e-01,\n 2.1862e-02, 2.6289e-01, 1.4750e-01, 2.4689e-01, 3.2552e-01,\n -3.2441e-01, -3.4585e-01, 3.3942e-01, -1.2287e-01, 3.3929e-01,\n -3.4600e-01, -2.5490e-01, -1.4810e-01, -2.4914e-01, -3.1616e-01,\n -2.7996e-01, 5.3910e-02, 8.3362e-02, -2.2756e-01, -3.4595e-01,\n 1.4570e-01, 2.2476e-01, 1.0500e-01, -3.5072e-01, -1.3940e-01,\n -1.2590e-01, -3.2186e-01, -2.0266e-01, 3.0027e-01, -5.6623e-02,\n -2.3789e-01, 3.3488e-01, 3.4398e-01, -3.1896e-01, 3.0663e-01,\n 2.8664e-01, 3.3766e-01, 6.4165e-02, -3.1826e-01, 2.6319e-01,\n -2.2919e-01, 3.5134e-01, 3.0663e-01, -1.3062e-01, -2.9913e-01,\n 1.3565e-01, -1.6022e-01, 2.2197e-01],\n [-2.1554e-02, -1.5210e-02, 3.7830e-02, 2.6736e-02, -2.1166e-02,\n -1.9468e-03, 8.0254e-02, -2.2102e-03, 2.5859e-03, -1.2345e-02,\n 2.7264e-02, 1.7788e-02, -1.5296e-02, 1.9181e-03, -1.3894e-02,\n 3.2927e-02, 8.5312e-03, 1.5223e-02, 2.1626e-02, -1.0207e-02,\n 3.5204e-02, 2.5883e-02, 2.4556e-02, 1.8632e-02, 6.5476e-05,\n -1.6241e-02, 7.4333e-03, 5.2232e-02, -1.6711e-02, 3.2708e-02,\n -1.0692e-02, -1.4305e-02, 1.3133e-02, 2.7117e-02, -1.4753e-02,\n 4.8611e-02, 1.4527e-02, 1.8879e-02, -1.4481e-02, 1.4897e-02,\n 1.1238e-02, -1.2450e-02, -1.6202e-02, 4.9784e-03, 1.2289e-02,\n 1.9180e-02, 1.4027e-02, -1.4833e-02, 1.6427e-02, 1.0015e-02,\n 5.9121e-02, 5.6994e-02, -3.0438e-02, -1.1613e-02, 1.1159e-02,\n -4.3374e-02, 2.7599e-02, -2.2615e-02, 1.4398e-02, 3.0019e-02,\n 1.0136e-02, 1.9818e-02, -1.4665e-02, -2.6874e-02, -1.8996e-02,\n -9.1702e-03, -1.3508e-02, -4.1096e-02, -2.9625e-02, 5.2459e-02,\n -4.2368e-02, -1.4232e-02, -1.5201e-02, 4.9654e-02, -1.1673e-02,\n 5.3639e-02, 1.8897e-02, -1.4671e-02, 1.5038e-02, 2.0248e-02,\n 2.6686e-02, -3.5407e-02, -1.6679e-02, -3.0283e-02, -2.9321e-02,\n 1.8608e-02, 1.5144e-02, -1.2829e-02, 8.7272e-03, -1.8822e-02,\n 1.6827e-02, 3.6772e-03, 5.2391e-03, 1.7730e-02, 9.1419e-03,\n 6.0197e-02, -5.2879e-03, 1.0835e-02, -1.6823e-03, 1.7445e-02,\n -2.1828e-02, -2.4577e-02, -2.9082e-02, 1.5298e-02, 4.3386e-02,\n -4.2322e-04, 6.9954e-02, -9.6678e-03, -3.8367e-02, 1.6948e-02,\n 3.5717e-02, -1.7567e-02, -3.4780e-02, 1.2462e-02, -1.0854e-02,\n -1.8944e-02, -2.4447e-02, 1.1157e-02, 1.2722e-02, -3.6463e-02,\n 1.8334e-02, -1.6749e-02, -6.1643e-02, 6.6864e-03, 3.9645e-03,\n -4.8894e-03, 4.2964e-03, 2.5981e-03]], device='mps:0')\nGrad: after - tensor([[ 0.1867, -0.2881],\n [-0.0076, 0.0535]], device='mps:0')\nGrad: after - tensor([ 0.3526, -0.0147], device='mps:0')\nGrad: after - tensor([ 0.3526, -0.0147], device='mps:0')\nAre all parameters equal? False\n</code></pre>\n<p>Thanks again for your help!</p>",252 "post_number": 5,253 "post_type": 1,254 "posts_count": 17,255 "updated_at": "2023-07-26T07:48:18.084Z",256 "reply_count": 1,257 "reply_to_post_number": 4,258 "quote_count": 1,259 "incoming_link_count": 2,260 "reads": 28,261 "readers_count": 27,262 "score": 20.6,263 "yours": false,264 "topic_id": 184989,265 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",266 "display_username": "Sebastian Hurubaru",267 "primary_group_name": null,268 "flair_name": null,269 "flair_url": null,270 "flair_bg_color": null,271 "flair_color": null,272 "flair_group_id": null,273 "badges_granted": [],274 "version": 1,275 "can_edit": false,276 "can_delete": false,277 "can_recover": false,278 "can_see_hidden_post": false,279 "can_wiki": false,280 "read": true,281 "user_title": null,282 "bookmarked": false,283 "actions_summary": [],284 "moderator": false,285 "admin": false,286 "staff": false,287 "user_id": 68141,288 "hidden": false,289 "trust_level": 1,290 "deleted_at": null,291 "user_deleted": false,292 "edit_reason": null,293 "can_view_edit_history": true,294 "wiki": false,295 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/5",296 "can_accept_answer": false,297 "can_unaccept_answer": false,298 "accepted_answer": false,299 "topic_accepted_answer": true300 },301 {302 "id": 411459,303 "name": "",304 "username": "ptrblck",305 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",306 "created_at": "2023-07-26T07:58:24.714Z",307 "cooked": "<p>OK, so it seems the gradients are calculated and the parameters updated.<br>\nIt’s still unclear why your model isn’t learning anything. Could you compare the gradient magnitudes between PyTorch 1.13.1 and 2.0.1?</p>",308 "post_number": 6,309 "post_type": 1,310 "posts_count": 17,311 "updated_at": "2023-07-26T07:58:24.714Z",312 "reply_count": 1,313 "reply_to_post_number": 5,314 "quote_count": 0,315 "incoming_link_count": 0,316 "reads": 24,317 "readers_count": 23,318 "score": 9.8,319 "yours": false,320 "topic_id": 184989,321 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",322 "display_username": "",323 "primary_group_name": null,324 "flair_name": null,325 "flair_url": null,326 "flair_bg_color": null,327 "flair_color": null,328 "flair_group_id": null,329 "badges_granted": [],330 "version": 1,331 "can_edit": false,332 "can_delete": false,333 "can_recover": false,334 "can_see_hidden_post": false,335 "can_wiki": false,336 "read": true,337 "user_title": "",338 "reply_to_user": {339 "id": 68141,340 "username": "SebastianHurubaru",341 "name": "Sebastian Hurubaru",342 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"343 },344 "bookmarked": false,345 "actions_summary": [],346 "moderator": true,347 "admin": true,348 "staff": true,349 "user_id": 3534,350 "hidden": false,351 "trust_level": 2,352 "deleted_at": null,353 "user_deleted": false,354 "edit_reason": null,355 "can_view_edit_history": true,356 "wiki": false,357 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/6",358 "can_accept_answer": false,359 "can_unaccept_answer": false,360 "accepted_answer": false,361 "topic_accepted_answer": true362 },363 {364 "id": 411484,365 "name": "Sebastian Hurubaru",366 "username": "SebastianHurubaru",367 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",368 "created_at": "2023-07-26T11:23:00.817Z",369 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> so the L2 distance between the gradients of the two versions, after the first time the <code>backward()</code> gets called is around <code>0.45</code>.</p>",370 "post_number": 7,371 "post_type": 1,372 "posts_count": 17,373 "updated_at": "2023-07-26T11:23:00.817Z",374 "reply_count": 1,375 "reply_to_post_number": 6,376 "quote_count": 0,377 "incoming_link_count": 1,378 "reads": 23,379 "readers_count": 22,380 "score": 14.6,381 "yours": false,382 "topic_id": 184989,383 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",384 "display_username": "Sebastian Hurubaru",385 "primary_group_name": null,386 "flair_name": null,387 "flair_url": null,388 "flair_bg_color": null,389 "flair_color": null,390 "flair_group_id": null,391 "badges_granted": [],392 "version": 1,393 "can_edit": false,394 "can_delete": false,395 "can_recover": false,396 "can_see_hidden_post": false,397 "can_wiki": false,398 "read": true,399 "user_title": null,400 "reply_to_user": {401 "id": 3534,402 "username": "ptrblck",403 "name": "",404 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"405 },406 "bookmarked": false,407 "actions_summary": [],408 "moderator": false,409 "admin": false,410 "staff": false,411 "user_id": 68141,412 "hidden": false,413 "trust_level": 1,414 "deleted_at": null,415 "user_deleted": false,416 "edit_reason": null,417 "can_view_edit_history": true,418 "wiki": false,419 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/7",420 "can_accept_answer": false,421 "can_unaccept_answer": false,422 "accepted_answer": false,423 "topic_accepted_answer": true424 },425 {426 "id": 411526,427 "name": "",428 "username": "ptrblck",429 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",430 "created_at": "2023-07-26T17:35:44.812Z",431 "cooked": "<p>I’m unsure what norm values would be expected, but I also see that you are using the <code>mps</code> backend.<br>\nDo you see the same behavior in any other backend (e.g. CPU or CUDA)?</p>",432 "post_number": 8,433 "post_type": 1,434 "posts_count": 17,435 "updated_at": "2023-07-26T17:35:44.812Z",436 "reply_count": 1,437 "reply_to_post_number": 7,438 "quote_count": 0,439 "incoming_link_count": 1,440 "reads": 23,441 "readers_count": 22,442 "score": 14.6,443 "yours": false,444 "topic_id": 184989,445 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",446 "display_username": "",447 "primary_group_name": null,448 "flair_name": null,449 "flair_url": null,450 "flair_bg_color": null,451 "flair_color": null,452 "flair_group_id": null,453 "badges_granted": [],454 "version": 1,455 "can_edit": false,456 "can_delete": false,457 "can_recover": false,458 "can_see_hidden_post": false,459 "can_wiki": false,460 "read": true,461 "user_title": "",462 "reply_to_user": {463 "id": 68141,464 "username": "SebastianHurubaru",465 "name": "Sebastian Hurubaru",466 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"467 },468 "bookmarked": false,469 "actions_summary": [],470 "moderator": true,471 "admin": true,472 "staff": true,473 "user_id": 3534,474 "hidden": false,475 "trust_level": 2,476 "deleted_at": null,477 "user_deleted": false,478 "edit_reason": null,479 "can_view_edit_history": true,480 "wiki": false,481 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/8",482 "can_accept_answer": false,483 "can_unaccept_answer": false,484 "accepted_answer": false,485 "topic_accepted_answer": true486 },487 {488 "id": 411594,489 "name": "Sebastian Hurubaru",490 "username": "SebastianHurubaru",491 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",492 "created_at": "2023-07-27T05:10:37.113Z",493 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> yes, the same behavior on Nvidia and CPU devices.</p>",494 "post_number": 9,495 "post_type": 1,496 "posts_count": 17,497 "updated_at": "2023-07-27T05:10:37.113Z",498 "reply_count": 1,499 "reply_to_post_number": 8,500 "quote_count": 0,501 "incoming_link_count": 1,502 "reads": 20,503 "readers_count": 19,504 "score": 14.0,505 "yours": false,506 "topic_id": 184989,507 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",508 "display_username": "Sebastian Hurubaru",509 "primary_group_name": null,510 "flair_name": null,511 "flair_url": null,512 "flair_bg_color": null,513 "flair_color": null,514 "flair_group_id": null,515 "badges_granted": [],516 "version": 1,517 "can_edit": false,518 "can_delete": false,519 "can_recover": false,520 "can_see_hidden_post": false,521 "can_wiki": false,522 "read": true,523 "user_title": null,524 "reply_to_user": {525 "id": 3534,526 "username": "ptrblck",527 "name": "",528 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"529 },530 "bookmarked": false,531 "actions_summary": [],532 "moderator": false,533 "admin": false,534 "staff": false,535 "user_id": 68141,536 "hidden": false,537 "trust_level": 1,538 "deleted_at": null,539 "user_deleted": false,540 "edit_reason": null,541 "can_view_edit_history": true,542 "wiki": false,543 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/9",544 "can_accept_answer": false,545 "can_unaccept_answer": false,546 "accepted_answer": false,547 "topic_accepted_answer": true548 },549 {550 "id": 411595,551 "name": "",552 "username": "ptrblck",553 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",554 "created_at": "2023-07-27T05:26:00.177Z",555 "cooked": "<p>Thank for checking. In this case I would start removing parts of the model until e.g. a single layer is used to check if you can train it at all. I haven’t seen this kind of issue before and since your parameters are updated, I guess something else might block the training.</p>",556 "post_number": 10,557 "post_type": 1,558 "posts_count": 17,559 "updated_at": "2023-07-27T05:26:00.177Z",560 "reply_count": 1,561 "reply_to_post_number": 9,562 "quote_count": 0,563 "incoming_link_count": 0,564 "reads": 19,565 "readers_count": 18,566 "score": 8.8,567 "yours": false,568 "topic_id": 184989,569 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",570 "display_username": "",571 "primary_group_name": null,572 "flair_name": null,573 "flair_url": null,574 "flair_bg_color": null,575 "flair_color": null,576 "flair_group_id": null,577 "badges_granted": [],578 "version": 1,579 "can_edit": false,580 "can_delete": false,581 "can_recover": false,582 "can_see_hidden_post": false,583 "can_wiki": false,584 "read": true,585 "user_title": "",586 "reply_to_user": {587 "id": 68141,588 "username": "SebastianHurubaru",589 "name": "Sebastian Hurubaru",590 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"591 },592 "bookmarked": false,593 "actions_summary": [],594 "moderator": true,595 "admin": true,596 "staff": true,597 "user_id": 3534,598 "hidden": false,599 "trust_level": 2,600 "deleted_at": null,601 "user_deleted": false,602 "edit_reason": null,603 "can_view_edit_history": true,604 "wiki": false,605 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/10",606 "can_accept_answer": false,607 "can_unaccept_answer": false,608 "accepted_answer": false,609 "topic_accepted_answer": true610 },611 {612 "id": 411650,613 "name": "Sebastian Hurubaru",614 "username": "SebastianHurubaru",615 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",616 "created_at": "2023-07-27T11:33:24.613Z",617 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> so the model has just two RNN layers.<br>\nSwitch to just one layer and we get the same bad results in PyTorch 2.0.1 while in 1.13.1 the performance goes from ~0.9 to ~0.75.</p>",618 "post_number": 11,619 "post_type": 1,620 "posts_count": 17,621 "updated_at": "2023-07-27T11:33:24.613Z",622 "reply_count": 1,623 "reply_to_post_number": 10,624 "quote_count": 0,625 "incoming_link_count": 2,626 "reads": 17,627 "readers_count": 16,628 "score": 18.4,629 "yours": false,630 "topic_id": 184989,631 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",632 "display_username": "Sebastian Hurubaru",633 "primary_group_name": null,634 "flair_name": null,635 "flair_url": null,636 "flair_bg_color": null,637 "flair_color": null,638 "flair_group_id": null,639 "badges_granted": [],640 "version": 1,641 "can_edit": false,642 "can_delete": false,643 "can_recover": false,644 "can_see_hidden_post": false,645 "can_wiki": false,646 "read": true,647 "user_title": null,648 "reply_to_user": {649 "id": 3534,650 "username": "ptrblck",651 "name": "",652 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"653 },654 "bookmarked": false,655 "actions_summary": [],656 "moderator": false,657 "admin": false,658 "staff": false,659 "user_id": 68141,660 "hidden": false,661 "trust_level": 1,662 "deleted_at": null,663 "user_deleted": false,664 "edit_reason": null,665 "can_view_edit_history": true,666 "wiki": false,667 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/11",668 "can_accept_answer": false,669 "can_unaccept_answer": false,670 "accepted_answer": false,671 "topic_accepted_answer": true672 },673 {674 "id": 411686,675 "name": "",676 "username": "ptrblck",677 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",678 "created_at": "2023-07-27T16:26:44.502Z",679 "cooked": "<p>In that case I would assume it should be easy to post a minimal and executable code snippet reproducing the training stagnation, which we could use to reproduce and debug it.</p>",680 "post_number": 12,681 "post_type": 1,682 "posts_count": 17,683 "updated_at": "2023-07-27T16:27:04.412Z",684 "reply_count": 1,685 "reply_to_post_number": 11,686 "quote_count": 0,687 "incoming_link_count": 0,688 "reads": 16,689 "readers_count": 15,690 "score": 8.2,691 "yours": false,692 "topic_id": 184989,693 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",694 "display_username": "",695 "primary_group_name": null,696 "flair_name": null,697 "flair_url": null,698 "flair_bg_color": null,699 "flair_color": null,700 "flair_group_id": null,701 "badges_granted": [],702 "version": 1,703 "can_edit": false,704 "can_delete": false,705 "can_recover": false,706 "can_see_hidden_post": false,707 "can_wiki": false,708 "read": true,709 "user_title": "",710 "reply_to_user": {711 "id": 68141,712 "username": "SebastianHurubaru",713 "name": "Sebastian Hurubaru",714 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"715 },716 "bookmarked": false,717 "actions_summary": [],718 "moderator": true,719 "admin": true,720 "staff": true,721 "user_id": 3534,722 "hidden": false,723 "trust_level": 2,724 "deleted_at": null,725 "user_deleted": false,726 "edit_reason": null,727 "can_view_edit_history": true,728 "wiki": false,729 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/12",730 "can_accept_answer": false,731 "can_unaccept_answer": false,732 "accepted_answer": false,733 "topic_accepted_answer": true734 },735 {736 "id": 411695,737 "name": "Sebastian Hurubaru",738 "username": "SebastianHurubaru",739 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",740 "created_at": "2023-07-27T18:17:17.260Z",741 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> absolutely. It is available <a href=\"https://gist.github.com/SebastianHurubaru/335eab6e9a89ce62dec73af16e983554\" rel=\"noopener nofollow ugc\">here</a>.</p>\n<p>The sample contains various conda environment files depending on your hardware (Mac/CPU and Nvidia GPU) for each of the PyTorch version 1.13.1 and 2.0.1.</p>\n<p>To run it on CPU the following command should be executed: <code>python main.py --num_shot 1 --num_classes 2</code>.</p>\n<p>Thanks again for all your help so far <a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> !</p>",742 "post_number": 13,743 "post_type": 1,744 "posts_count": 17,745 "updated_at": "2023-07-27T18:17:17.260Z",746 "reply_count": 1,747 "reply_to_post_number": 12,748 "quote_count": 0,749 "incoming_link_count": 2,750 "reads": 16,751 "readers_count": 15,752 "score": 18.2,753 "yours": false,754 "topic_id": 184989,755 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",756 "display_username": "Sebastian Hurubaru",757 "primary_group_name": null,758 "flair_name": null,759 "flair_url": null,760 "flair_bg_color": null,761 "flair_color": null,762 "flair_group_id": null,763 "badges_granted": [],764 "version": 1,765 "can_edit": false,766 "can_delete": false,767 "can_recover": false,768 "can_see_hidden_post": false,769 "can_wiki": false,770 "link_counts": [771 {772 "url": "https://gist.github.com/SebastianHurubaru/335eab6e9a89ce62dec73af16e983554",773 "internal": false,774 "reflection": false,775 "title": "Big accuracy differences between PyTorch 1.13.1 and 2.0.1! · GitHub",776 "clicks": 5777 }778 ],779 "read": true,780 "user_title": null,781 "reply_to_user": {782 "id": 3534,783 "username": "ptrblck",784 "name": "",785 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"786 },787 "bookmarked": false,788 "actions_summary": [],789 "moderator": false,790 "admin": false,791 "staff": false,792 "user_id": 68141,793 "hidden": false,794 "trust_level": 1,795 "deleted_at": null,796 "user_deleted": false,797 "edit_reason": null,798 "can_view_edit_history": true,799 "wiki": false,800 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/13",801 "can_accept_answer": false,802 "can_unaccept_answer": false,803 "accepted_answer": false,804 "topic_accepted_answer": true805 },806 {807 "id": 411702,808 "name": "",809 "username": "ptrblck",810 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",811 "created_at": "2023-07-27T19:34:36.819Z",812 "cooked": "<p>Thanks! Unfortunately, your code isn’t executable without downloading an unknown dataset from an unknown source.<br>\nThe comments in your model also don’t seem to be valid as using any value combination for <code>[B, K+1, N, 784]</code> for the data and <code>[B, K+1, N, N]</code> results in shape mismatches, so could you let me know what the expected shapes are?</p>",813 "post_number": 14,814 "post_type": 1,815 "posts_count": 17,816 "updated_at": "2023-07-27T19:34:36.819Z",817 "reply_count": 0,818 "reply_to_post_number": 13,819 "quote_count": 0,820 "incoming_link_count": 1,821 "reads": 16,822 "readers_count": 15,823 "score": 8.2,824 "yours": false,825 "topic_id": 184989,826 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",827 "display_username": "",828 "primary_group_name": null,829 "flair_name": null,830 "flair_url": null,831 "flair_bg_color": null,832 "flair_color": null,833 "flair_group_id": null,834 "badges_granted": [],835 "version": 1,836 "can_edit": false,837 "can_delete": false,838 "can_recover": false,839 "can_see_hidden_post": false,840 "can_wiki": false,841 "read": true,842 "user_title": "",843 "reply_to_user": {844 "id": 68141,845 "username": "SebastianHurubaru",846 "name": "Sebastian Hurubaru",847 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"848 },849 "bookmarked": false,850 "actions_summary": [],851 "moderator": true,852 "admin": true,853 "staff": true,854 "user_id": 3534,855 "hidden": false,856 "trust_level": 2,857 "deleted_at": null,858 "user_deleted": false,859 "edit_reason": null,860 "can_view_edit_history": true,861 "wiki": false,862 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/14",863 "can_accept_answer": false,864 "can_unaccept_answer": false,865 "accepted_answer": false,866 "topic_accepted_answer": true867 },868 {869 "id": 411707,870 "name": "Sebastian Hurubaru",871 "username": "SebastianHurubaru",872 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",873 "created_at": "2023-07-27T19:52:05.194Z",874 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a> we are using the Omniglot dataset, i.e. the transpose of MNIST, as per the <a href=\"https://arxiv.org/pdf/1605.06065.pdf\" rel=\"noopener nofollow ugc\">MANN</a> paper we are implementing in the code sample. If this poses a security threat for you, could switch to MNIST, though it will take some time.</p>\n<p>The code should definitely run as it is, so not sure what you mean with the comments related to the shapes. Have you executed the code and received any errors?</p>",875 "post_number": 15,876 "post_type": 1,877 "posts_count": 17,878 "updated_at": "2023-07-27T19:52:05.194Z",879 "reply_count": 1,880 "reply_to_post_number": null,881 "quote_count": 0,882 "incoming_link_count": 7,883 "reads": 15,884 "readers_count": 14,885 "score": 43.0,886 "yours": false,887 "topic_id": 184989,888 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",889 "display_username": "Sebastian Hurubaru",890 "primary_group_name": null,891 "flair_name": null,892 "flair_url": null,893 "flair_bg_color": null,894 "flair_color": null,895 "flair_group_id": null,896 "badges_granted": [],897 "version": 1,898 "can_edit": false,899 "can_delete": false,900 "can_recover": false,901 "can_see_hidden_post": false,902 "can_wiki": false,903 "link_counts": [904 {905 "url": "https://arxiv.org/pdf/1605.06065.pdf",906 "internal": false,907 "reflection": false,908 "clicks": 3909 }910 ],911 "read": true,912 "user_title": null,913 "bookmarked": false,914 "actions_summary": [],915 "moderator": false,916 "admin": false,917 "staff": false,918 "user_id": 68141,919 "hidden": false,920 "trust_level": 1,921 "deleted_at": null,922 "user_deleted": false,923 "edit_reason": null,924 "can_view_edit_history": true,925 "wiki": false,926 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/15",927 "can_accept_answer": false,928 "can_unaccept_answer": false,929 "accepted_answer": false,930 "topic_accepted_answer": true931 },932 {933 "id": 411942,934 "name": "Arul",935 "username": "InnovArul",936 "avatar_template": "/user_avatar/discuss.pytorch.org/innovarul/{size}/5282_2.png",937 "created_at": "2023-07-30T13:20:49.990Z",938 "cooked": "<p>Not sure if you already spotted the issue.<br>\nThanks for providing the code to reproduce.</p>\n<p>I used Windows 11 to run all the combinations,</p>\n<ul>\n<li>Pytorch 1.13 + cuda (<code>environment_113_cuda.yml</code>)</li>\n<li>Pytorch 2.0.1 + cuda (<code>environment_201_cuda.yml</code>),</li>\n<li>Pytorch 2.1.0 + cuda (<code>pytorch_nightly</code>)</li>\n</ul>\n<p>I faced an image normalization issue due to <code>imageio</code> version mismatch (<code>environment_113_cuda</code> = <code>imageio 2.19.3</code>, <code>environment_201_cuda</code> = <code>imageio 2.31.1</code>).</p>\n<p>In your code, the normalization is done as follows:</p>\n<pre><code class=\"lang-auto\"> image = imageio.imread(filename)\n image = image.reshape([dim_input])\n image = image.astype(np.float32) / 255.0\n</code></pre>\n<p>In <code>imageio 2.19.3</code>, the min and max values are [0, 255].<br>\nHowever in <code>imageio 2.31.1</code>, the min and max values are [0., 1.]. Hence <code>(image / 255.)</code> makes the image to (almost) 0 in the pytorch 2.0.1 conda environment.</p>\n<p>After handling this normalization correctly, all the pytorch versions give similar performance (atleast in my experiments).</p>\n<pre><code class=\"lang-auto\"> image = image / image.max()\n</code></pre>\n<p>I am not sure if this is the same issue that you are facing though.</p>",939 "post_number": 16,940 "post_type": 1,941 "posts_count": 17,942 "updated_at": "2023-07-30T13:21:23.064Z",943 "reply_count": 1,944 "reply_to_post_number": 15,945 "quote_count": 0,946 "incoming_link_count": 31,947 "reads": 12,948 "readers_count": 11,949 "score": 252.4,950 "yours": false,951 "topic_id": 184989,952 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",953 "display_username": "Arul",954 "primary_group_name": null,955 "flair_name": null,956 "flair_url": null,957 "flair_bg_color": null,958 "flair_color": null,959 "flair_group_id": null,960 "badges_granted": [],961 "version": 1,962 "can_edit": false,963 "can_delete": false,964 "can_recover": false,965 "can_see_hidden_post": false,966 "can_wiki": false,967 "read": true,968 "user_title": "",969 "reply_to_user": {970 "id": 68141,971 "username": "SebastianHurubaru",972 "name": "Sebastian Hurubaru",973 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png"974 },975 "bookmarked": false,976 "actions_summary": [977 {978 "id": 2,979 "count": 2980 }981 ],982 "moderator": false,983 "admin": false,984 "staff": false,985 "user_id": 998,986 "hidden": false,987 "trust_level": 2,988 "deleted_at": null,989 "user_deleted": false,990 "edit_reason": null,991 "can_view_edit_history": true,992 "wiki": false,993 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/16",994 "can_accept_answer": false,995 "can_unaccept_answer": false,996 "accepted_answer": true,997 "topic_accepted_answer": true998 },999 {1000 "id": 412025,1001 "name": "Sebastian Hurubaru",1002 "username": "SebastianHurubaru",1003 "avatar_template": "/user_avatar/discuss.pytorch.org/sebastianhurubaru/{size}/62592_2.png",1004 "created_at": "2023-07-31T08:06:44.924Z",1005 "cooked": "<aside class=\"quote no-group\" data-username=\"InnovArul\" data-post=\"16\" data-topic=\"184989\">\n<div class=\"title\">\n<div class=\"quote-controls\"></div>\n<img loading=\"lazy\" alt=\"\" width=\"24\" height=\"24\" src=\"https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/innovarul/48/5282_2.png\" class=\"avatar\"> InnovArul:</div>\n<blockquote>\n<p><code>image.max()</code></p>\n</blockquote>\n</aside>\n<p><a class=\"mention\" href=\"/u/innovarul\">@InnovArul</a> thanks a lot! That was it!</p>",1006 "post_number": 17,1007 "post_type": 1,1008 "posts_count": 17,1009 "updated_at": "2023-07-31T08:06:44.924Z",1010 "reply_count": 0,1011 "reply_to_post_number": 16,1012 "quote_count": 1,1013 "incoming_link_count": 2,1014 "reads": 11,1015 "readers_count": 10,1016 "score": 12.2,1017 "yours": false,1018 "topic_id": 184989,1019 "topic_slug": "big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1",1020 "display_username": "Sebastian Hurubaru",1021 "primary_group_name": null,1022 "flair_name": null,1023 "flair_url": null,1024 "flair_bg_color": null,1025 "flair_color": null,1026 "flair_group_id": null,1027 "badges_granted": [],1028 "version": 1,1029 "can_edit": false,1030 "can_delete": false,1031 "can_recover": false,1032 "can_see_hidden_post": false,1033 "can_wiki": false,1034 "read": true,1035 "user_title": null,1036 "bookmarked": false,1037 "actions_summary": [],1038 "moderator": false,1039 "admin": false,1040 "staff": false,1041 "user_id": 68141,1042 "hidden": false,1043 "trust_level": 1,1044 "deleted_at": null,1045 "user_deleted": false,1046 "edit_reason": null,1047 "can_view_edit_history": true,1048 "wiki": false,1049 "post_url": "/t/big-accuracy-differences-between-pytorch-1-13-1-and-2-0-1/184989/17",1050 "can_accept_answer": false,1051 "can_unaccept_answer": false,1052 "accepted_answer": false,1053 "topic_accepted_answer": true1054 }1055 ],1056 "stream": [1057 411315,1058 411317,1059 411385,1060 411396,1061 411457,1062 411459,1063 411484,1064 411526,1065 411594,1066 411595,1067 411650,1068 411686,1069 411695,1070 411702,1071 411707,1072 411942,1073 4120251074 ]1075 },1076 "timeline_lookup": [1077 [1078 1,1079 8231080 ],1081 [1082 5,1083 8221084 ],1085 [1086 11,1087 8211088 ],1089 [1090 16,1091 8181092 ],1093 [1094 17,1095 8171096 ]1097 ],1098 "suggested_topics": [1099 {1100 "fancy_title": "Torch version + cuda stable compatibility details:",1101 "id": 217149,1102 "title": "Torch version + cuda stable compatibility details:",1103 "slug": "torch-version-cuda-stable-compatibility-details",1104 "posts_count": 3,1105 "reply_count": 1,1106 "highest_post_number": 3,1107 "image_url": null,1108 "created_at": "2025-02-25T14:21:17.809Z",1109 "last_posted_at": "2025-02-25T14:57:20.006Z",1110 "bumped": true,1111 "bumped_at": "2025-02-25T14:57:20.006Z",1112 "archetype": "regular",1113 "unseen": false,1114 "pinned": false,1115 "unpinned": null,1116 "visible": true,1117 "closed": false,1118 "archived": false,1119 "bookmarked": null,1120 "liked": null,1121 "tags_descriptions": {},1122 "like_count": 0,1123 "views": 129,1124 "category_id": 1,1125 "featured_link": null,1126 "has_accepted_answer": false,1127 "posters": [1128 {1129 "extras": "latest",1130 "description": "Original Poster, Most Recent Poster",1131 "user": {1132 "id": 82794,1133 "username": "Mohan_Krishna",1134 "name": "Mohan Krishna",1135 "avatar_template": "/user_avatar/discuss.pytorch.org/mohan_krishna/{size}/75758_2.png",1136 "trust_level": 11137 }1138 },1139 {1140 "extras": null,1141 "description": "Frequent Poster",1142 "user": {1143 "id": 3534,1144 "username": "ptrblck",1145 "name": "",1146 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1147 "admin": true,1148 "moderator": true,1149 "trust_level": 21150 }1151 }1152 ]1153 },1154 {1155 "fancy_title": "Correct SDPA’s attn_mask for Self-attention",1156 "id": 213077,1157 "title": "Correct SDPA's attn_mask for Self-attention",1158 "slug": "correct-sdpas-attn-mask-for-self-attention",1159 "posts_count": 1,1160 "reply_count": 0,1161 "highest_post_number": 1,1162 "image_url": null,1163 "created_at": "2024-11-17T14:32:43.904Z",1164 "last_posted_at": "2024-11-17T14:32:43.958Z",1165 "bumped": true,1166 "bumped_at": "2024-11-17T14:32:43.958Z",1167 "archetype": "regular",1168 "unseen": false,1169 "pinned": false,1170 "unpinned": null,1171 "visible": true,1172 "closed": false,1173 "archived": false,1174 "bookmarked": null,1175 "liked": null,1176 "tags_descriptions": {},1177 "like_count": 0,1178 "views": 204,1179 "category_id": 1,1180 "featured_link": null,1181 "has_accepted_answer": false,1182 "posters": [1183 {1184 "extras": "latest single",1185 "description": "Original Poster, Most Recent Poster",1186 "user": {1187 "id": 80966,1188 "username": "SotoJnthn",1189 "name": "Jonathan",1190 "avatar_template": "/user_avatar/discuss.pytorch.org/sotojnthn/{size}/74047_2.png",1191 "trust_level": 11192 }1193 }1194 ]1195 },1196 {1197 "fancy_title": "Nvrtc error with CUDA",1198 "id": 213804,1199 "title": "Nvrtc error with CUDA",1200 "slug": "nvrtc-error-with-cuda",